functions

#!/usr/bin/env bash

### ############################################################################

### openspace42 synthia bootstrap framework.
### [https://github.com/openspace42/synthia].
### v0.4.0.

dna_version="v0.3.14"

### ############################################################################

### 1 | synthia global functions

### ############################################################################

synthia-define_vars() {

	### define default variables.

	### 1 | synthia global variables.

	### ########################################################################

	export proj_name="aenigma" ## legacy fix
	export project_name="aenigma"
	export author_name="openspace42"
	export git_host="https://github.com"

	### ########################################################################

	### 2 | synthia project_specific variales.

	### ########################################################################

	#### backup / restore.

	#### #######################################################################

	#### set this to 'disabled' to disable local backups and only upload to S3
	#### [useful if your project handles large amounts of data for example].
	#### S3 must be set up by the user or else backups will not work at all.
	#### this is only the default, which will be suggested to the user during
	#### install, and it can be overridden.

	export backup_locally_mode_default="enabled"

	#### set this to 'disabled' to disable restore test during S3 backup.
	#### this is only the default, which will be suggested to the user during
	#### install, and it can be overridden.

	export backup_test_restore_mode_default="enabled"

	#### set this to 'n' if your project stores no data on end users' machines
	#### that could go lost during a re-install or update.

	export backup_prompt_during_install="y"

	#### #######################################################################

	### ########################################################################

	### 3 | define dna variables.

	dna-define_vars

	### 4 | define any additional custom variables.

	### ########################################################################

	#### set the paths to the cluster / nodes conf dirs.

	export conf_cluster_dir="${conf_dir}/cluster"
	export conf_cluster_nodes_dir="${conf_cluster_dir}/nodes"

	export conf_xmpp_dir="${conf_dir}/xmpp"

	#### set the path to the domains_db file.

	export domains_db_path="${conf_xmpp_dir}/domains_db"

	#### set the paths to the nodes_db files.

	export nodes_db_ipv4_path="${conf_cluster_nodes_dir}/nodes_db_ipv4"
	export nodes_db_ipv6_path="${conf_cluster_nodes_dir}/nodes_db_ipv6"
	export nodes_db_numbered_path="${conf_cluster_nodes_dir}/nodes_db_numbered"
	export nodes_db_online_path="${tmp_project_dir}/nodes_db_online"
	export nodes_db_initial_path="${tmp_project_dir}/nodes_db_initial"

	#### hardcode package versions for installation.

	export hardcoded_package_version_ejabberd="19.05"
	export hardcoded_package_version_ejabberd_extended="19.05-0"
	export hardcoded_package_version_ejabberd_fallback="19.05"
	export hardcoded_package_version_ejabberd_fallback_extended="19.05-0"

	export hardcoded_package_version_conversejs="v5.0.1"

	### ########################################################################

	### 5 | define final variables that depend on custom ones.

	### ########################################################################

	#### set default backup paths.
	#### [absolute, mark whether existing or generated].

	declare -gA backup_paths_default_array=(
		[/var/www/upload]="existing"
		[/pgdump]="generated"
	)

	### ########################################################################

}

#### bootstrap functions.

#### ###########################################################################

synthia-define_formatting() {

	r=$'\e[1;31m'
	g=$'\e[1;32m'
	l=$'\e[1;34m'
	m=$'\e[1;35m'
	y=$'\e[1;33m'
	o=$'\e[38;5;208m'
	c=$'\e[1;36m'
	n=$'\e[1;39m'
	x=$'\e[0m'
	b=$'\033[1m'

}

synthia-check_root() {

	if [[ $EUID -ne 0 ]]
	then
		echo "${r-}${b-}This script must be run as root.${x-}"
		echo
		echo "${b-}Exiting...${x-}"
		echo
		exit 1
	fi

}

synthia-download_dna() {

	clone_host="https://github.com"
	clone_author="openspace42"
	clone_name="dna"

	clone_base_dir="/root/${clone_author}"
	clone_dir="${clone_base_dir}/${clone_name}"

	mkdir -p "${clone_base_dir}"

	if [ "${custom_dna_version-null}" = "l" ]
	then

		export dna_selected_vers="v0.0.1-custom-local"

		echo "${r-}${b-}Sourcing | ${n-}local version${r-} | of | ${n-}dna${r-} | respecting any local alteration performed as per | ${n-}-d l${r-} |.${x-}"

		echo "${dna_selected_vers}" > "./version_installed"

	else

		if [ -d "${clone_dir}" ]
		then
			rm -r "${clone_dir:?}"
		fi

		( cd "${clone_base_dir}" && git clone "${clone_host}/${clone_author}/${clone_name}" && echo && cd "${clone_name}"

			. ./functions/snippets

			git config advice.detachedHead false

			### Define latest version

			export dna_latest_vers_final="$(semtag getfinal)"
			export dna_latest_vers_tagged="$(semtag getlast)"
			export dna_latest_vers_bl_edge="$(semtag getcurrent)"

		    if [ "${custom_dna_version-null}" = "b" ]
		    then

		        export dna_selected_vers="${dna_latest_vers_bl_edge}"

		        echo "${r-}${b-}Installing | ${n-}bleeding-edge version${r-} | of | ${n-}dna${r-} | up to latest git commit as per | ${n-}-d b${r-} |.${x-}"
				echo

				git fetch
		        git reset --hard origin/master

			elif [[ "${custom_dna_version-null}" == *"v"* ]]
			then

				export dna_selected_vers="${custom_dna_version}"

				echo "${c-}${b-}Installing | ${n-}version ${dna_selected_vers}${c-} | of | ${n-}dna${c-} | as per | ${n-}-d ${dna_selected_vers}${c-} |.${x-}"
				echo

				git checkout "tags/${dna_selected_vers}"

		    elif [ "${custom_dna_version-null}" = "t" ]
		    then

		        export dna_selected_vers="${dna_latest_vers_tagged}"

		        echo "${o-}${b-}Installing | ${n-}latest-tagged version [${dna_selected_vers}]${o-} | of | ${n-}dna${o-} | [independently of its stability type] as per | ${n-}-d t${o-} |.${x-}"
				echo

		        git checkout "tags/${dna_latest_vers_tagged}"

		    elif [ "${custom_dna_version-null}" = "f" ]
			then

		        export dna_selected_vers="${dna_latest_vers_final}"

				echo "${y-}${b-}Installing | ${n-}latest-final version [${dna_selected_vers}]${y-} | of | ${n-}dna${y-} | as per | ${n-}-d f${y-} |.${x-}"
				echo

		        git checkout "tags/${dna_latest_vers_final}"

			else

				export dna_selected_vers="${dna_version}"

				echo "${b-}Installing | ${g-}version [${dna_selected_vers}]${n-} | of | ${g-}dna${n-} |.${x-}"
				echo

				git checkout "tags/${dna_version}"

		    fi

			echo "${dna_selected_vers}" > "./version_installed"

		)

	fi

}

synthia-source_dna() {

	for f in /root/openspace42/dna/functions/*
	do
		. $f
	done

}

#### ###########################################################################

#### backup_restore_functions

synthia-backup() {

	### evaluate arguments.

	### ########################################################################

    while getopts ":lsri" arguments
	do
		case $arguments in
			l)	export backup_locally_only="y"
				;;
			s)	export backup_s3_only="y"
				;;
			r)	export backup_last_before_restore="y"
				;;
			i)	export backup_skip_install_complete_check="y"
				;;
			\?)	dna-echo -e "Invalid option | ${n-}-${OPTARG}${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
		        dna-exit
		        ;;
		esac
	done
    OPTIND=1

	if [ -n "${backup_last_before_restore-}" ]
	then

		if [ -n "${backup_s3_only-}" ]
		then

			dna-echo -e "Option | ${n-}backup_s3_only [-s]${r-} | is incompatible with option | ${n-}backup_last_before_restore [-r]${r-} |."
			dna-exit

		fi

		export backup_locally_only="y"

	fi

	### ########################################################################

	dna-backup_check_set_lockfile

	if ! dna-backup
	then

		dna-echo -e "backup function exited with errors."
		dna-echo -m "continuing anyway to perform post-backup operations if any and exit backup function gracefully."

		synthia-backup_post_operations

	fi

	dna-backup_cleanup

}

#### ###########################################################################

### ############################################################################

### 2 | synthia project-specific functions.

### ############################################################################

#### backup / restore.

#### ###########################################################################

synthia-backup_pre_operations() {

	### perform pre-backup operations.

	dna-echo_operation -h "performing pre-backup operations"

	### ########################################################################
	### ########################################################################

	dna-echo_operation -t

}

synthia-backup_post_operations() {

	dna-echo_operation -h "performing post-backup operations"

	### perform post-backup operations.

	### ########################################################################
	### ########################################################################

	dna-echo_operation -t

}

synthia-calculate_backup_size() {

	### ########################################################################

	### calculate backup_paths size.

	dna-calculate_backup_paths_size

	### ########################################################################

	### calculate any additional custom size.

	### ########################################################################

	#### calculate postgresql database size.

	dna-get_hostname

	patroni_ejabberd_db_size="$(PGPASSWORD="${patroni_postgres_password}" psql -h "${hostname}" -p "5432" -U "postgres" -t -c "select pg_database_size('ejabberd') / 1024 / 1024;" | xargs)"

	### ########################################################################

	### add it up.

	backup_total_size_estimate="$((${backup_paths_size_estimate} + ${patroni_ejabberd_db_size}))"

	### ########################################################################

}

synthia-backup_generate_additional_paths() {

	### generate backups for any additional custom item by populating their
	### respective backup_paths.

	dna-echo_operation -h "backing up any additional custom backup_path"

	### ########################################################################

	### create paths

	dna-backup_generate_additional_paths_dirs

	### ########################################################################

	### perform backups of every item individually

	### ########################################################################

	#### postgresql dump.

	pg_dump --dbname="postgresql://ejabberd:${patroni_ejabberd_password}@${public_ipv4}:5432/ejabberd" > "${tmp_project_backup_dir}/pgdump/ejabberd.pgsql"

	### ########################################################################

	dna-echo_operation -t

}

synthia-backup_locally() {

	dna-echo_operation -h "backing up locally"

	### generate backups for any additional custom item

	synthia-backup_generate_additional_paths

	### backup backup paths locally.

	dna-backup_locally_data_backup_paths

	### backup any additional custom item.

	### ########################################################################
	### ########################################################################

	dna-echo_operation -t

}

synthia-restore() {

	### evaluate arguments.

	### ########################################################################

    while getopts ":lsri" arguments
	do
		case $arguments in
			l)	export backup_locally_only="y"
				;;
			\?)	dna-echo -e "Invalid option | ${n-}-${OPTARG}${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
		        dna-exit
		        ;;
		esac
	done
    OPTIND=1

	### ########################################################################

	if ! dna-backup
	then

		dna-echo -e "restore function exited with errors."

	fi

}

restore_loop() {

	echo "${b-}1] Now restoring aenigma config directory...${x-}"
	echo

	rsync -aAXx --delete $restore_file_path/conf/ $conf_dir/

	echo "${b-}2] Now restoring ejabberd database...${x-}"
	echo

	if (( $(ps -ef | grep -v grep | grep ejabberd | wc -l) > 0 ))
	then
	        echo "${g-}${b-}ejabberd is already running. Continuing...${x-}"
	        echo
	else
	        echo "${r-}${b-}ejabberd NOT running. Starting it now...${x-}"
	        echo
	        echo "${b-}Now starting ejabberd ...${x-}"
	        /usr/sbin/service ejabberd start
	        echo
	        dna-wait -s "8" -d "Now allowing ejabberd to start up correctly..."
	        /usr/sbin/ejabberdctl
	        echo
	        sleep 1
	fi

	if (( $(ps -ef | grep -v grep | grep ejabberd | wc -l) > 0 ))
	then
	        cd $restore_file_path/data/
		cp ./ejabberd-mnesia-backup /var/lib/ejabberd/
	        /usr/sbin/ejabberdctl restore ejabberd-mnesia-backup
		rm /var/lib/ejabberd/ejabberd-mnesia-backup
	        echo "${b-}Successfully restored ejabberd database...${x-}"
	        echo
	        dna-wait -s "8" -d "Now allowing ejabberd sync the database..."
	else
	        echo "${r-}${b-}ERROR: ejabberd STILL NOT running. Unable to restore database from backup.${x-}"
	        echo
	        dna-exit
	fi

	echo "${b-}Now restoring ejabberd directories...${x-}"
	echo

	echo "${b-}Now stopping ejabberd for directory restore...${x-}"
	echo
	/usr/sbin/service ejabberd stop
	dna-wait -s "8" -d "Now allowing ejabberd to shut down correctly..."

	echo "${b-}3] Now syncing ejabberd directories...${x-}"
	echo

	rsync -aAXx --delete $restore_file_path/data/etc-ejabberd/ /opt/ejabberd/
	rsync -aAXx --delete $restore_file_path/data/var-lib-ejabberd/ /var/lib/ejabberd/

	aenigma-set_permissions

	echo "${b-}Now starting ejabberd...${x-}"
	/usr/sbin/service ejabberd start
	echo
	dna-wait -s "8" -d "Now allowing ejabberd to start up correctly..."
	/usr/sbin/ejabberdctl status
	echo
	sleep 1

	if (( $(ps -ef | grep -v grep | grep ejabberd | wc -l) > 0 ))
	then

	        echo "${g-}${b-}Restore complete!${x-}"
	        echo
	        echo "${b-}Now please verify your aenigma instance is running correctly and has everything you except it to.${x-}"
	        echo

	        read -p "${b-}Has your previous instance of aenigma been restored correctly? (Y/n): ${x-}" -n 1 -r
	        echo
	        if [[ ! $REPLY =~ ^[Nn]$ ]]
	        then
	                echo "${g-}${b-}Ok, all done!${x-}"
	                echo
	                if [ -f $conf_dir/blocks3backups ]
	                then
				rm $conf_dir/blocks3backups
	                        echo "${b-}Removing S3 backups lock since your restore has been successful.${x-}"
	                        echo
	                        echo "${r-}${b-}This machine will now pick up where the old one left off and start backing up again to S3 and will overwrite all previous data stored on S3.${x-}"
	                        echo
	                        echo "${b-}If you've changed your mind and are NOT yet sure this restore was completely successful, simply:${x-}"
	                        echo
	                        echo "${b-} | touch $conf_dir/blocks3backups |"
	                        echo
	                        echo "${b-}And remember to remove it as soon as you're done verifying this restored instance to then resume backups to S3!${x-}"
	                        echo
	                        echo "${g-}${b-}All done for now.${x-}"
	                        echo
	                fi
	        else
	                echo
	                echo "${b-}Sorry about that, please collect all evidence you can find so and simply file an issue report at:${x-}"
		        	echo
		        	echo "${b-} | ${git_host}/${author_name}/${project_name}/issues |${x-}"
		        	echo
		        	echo "${b-}Thank you in advance!${x-}"
	                echo
	        fi

	else

	        echo "${r-}${b-}ERROR: ejabberd not running after restore. Please check ejabberd directories.${x-}"
	        echo
	        echo "${b-}Please collect all evidence you can find so and simply file an issue report at:${x-}"
	        echo
	        echo "${b-} | https://github.com/openspace42/aenigma/issues |${x-}"
	        echo
	        echo "${b-}Thank you in advance!${x-}"
	        echo
	        echo "${b-}Exiting...${x-}"
	        echo
	        exit

	fi

	if [ -d "$restores_dir/local/" ]
	then
		rm -r "$restores_dir/local/"
	fi

	if [ -d "/root/${project_name}-restore/" ]
	then
		rm -r "/root/${project_name}-restore/"
	fi

	if [ -d "$tmpdir" ]
	then
		rm -r "${tmpdir:?}"
	fi

	if [ $bootstrapmode = "on" ]
	then
		echo "${b-}Now that we're finished with the initial restore, it's time to re-run the install script and get this machine up and running with your old instance's files and settings${x-}"
		echo
		echo "${b-}Running install script now.${x-}"
		echo
		echo "${r-}${b-}Should you exit the install script during its runtime, you can re-run it manually with:${x-}"
		echo
		echo "${b-} | bash $sourcedir/install.sh | ${x-}"
		cd
		bash $sourcedir/install.sh
	fi

}

#### ###########################################################################

aenigma-display_logo() {

	### add delay for previous outputs.

	sleep 1

	### determine terminal window size.

	columns_number="$(tput cols)"
	rows_number="$(tput lines)"

	### check that the terminal window is large enough to display the logo.

	if [ "${columns_number}" -ge "93" ] && [ "${rows_number}" -ge "24" ]
	then

	    ### determine terminal window horizontal center.

	    columns_median="$(echo $((columns_number / 2)))"

	    ### the aenigma logo is 45 pixels and therefore 90 columns wide.

	    base_column="$(echo $((columns_median - 45)))"

		### determine terminal window vertical center.

	    rows_median="$(echo $((rows_number / 2)))"

	    ### the aenigma logo is 17 rows tall.

	    base_row="$(echo $((rows_median - 9)))"

		### determine bottom padding.

		bottom_padding="$(echo $((base_row - 2)))"

	    ### set colors.

	    b=$'\e[38;5;16m'
	    w=$'\e[38;5;15m'

	    l=$'\e[38;5;33m'

	    x=$'\e[0m'
	    z=$'\e[8m'

	    ### set white base for all lines.

	    set p02{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p03{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p04{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p05{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p06{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p07{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p08{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p09{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p10{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p11{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p12{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p13{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p14{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p15{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p16{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item
	    set p17{01..45} && array="$@" && for item in $array; do eval $item='$w'; done && unset array item

	    ### line 03.

	    eval p0319='$b'

	    ### line 06.

	    eval p06{03..09}='$l'
	    eval p06{11..17}='$b'
	    eval p0619='$b'
	    eval p06{21..27}='$b'
	    eval p06{29..35}='$b'
	    eval p06{38..43}='$b'

	    ### lines {07..08}.

		eval p{07..08}{06..09}='$l'
	    eval p{07..08}11='$b'
	    eval p{07..08}17='$b'
	    eval p{07..08}19='$b'
	    eval p{07..08}21='$b'
	    eval p{07..08}27='$b'
	    eval p{07..08}29='$b'
	    eval p{07..08}32='$b'
	    eval p{07..08}35='$b'
	    eval p{07..08}43='$b'

	    ### line 09.

	    eval p09{03..09}='$l'
	    eval p0911='$b'
	    eval p0917='$b'
	    eval p0919='$b'
	    eval p0921='$b'
	    eval p0927='$b'
	    eval p0929='$b'
	    eval p0932='$b'
	    eval p0935='$b'
	    eval p09{37..43}='$b'

	    ### lines {10..11}.

		eval p{10..11}{03..06}='$l'
	    eval p{10..11}11='$b'
	    eval p{10..11}17='$b'
	    eval p{10..11}19='$b'
	    eval p{10..11}21='$b'
	    eval p{10..11}27='$b'
	    eval p{10..11}29='$b'
	    eval p{10..11}32='$b'
	    eval p{10..11}35='$b'
	    eval p{10..11}37='$b'
	    eval p{10..11}43='$b'

	    ### line 12.

	    eval p12{03..09}='$l'
	    eval p1211='$b'
	    eval p1217='$b'
	    eval p1219='$b'
	    eval p12{21..27}='$b'
	    eval p1229='$b'
	    eval p1232='$b'
	    eval p1235='$b'
	    eval p12{37..43}='$b'

	    ### lines {13..14}.

	    eval p{13..14}27='$b'

	    ### line 15.

	    eval p15{22..27}='$b'

	    ### set borders.

	    set p01{01..45} && array="$@" && for item in $array; do eval $item='$b'; done && unset array item
	    set p17{01..45} && array="$@" && for item in $array; do eval $item='$b'; done && unset array item
	    eval p{01..17}01='$b'
	    eval p{01..17}45='$b'

		### clear.

		clear

		### print top padding.

		yes '' | sed ${base_row}q

	    ### print.

	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p01{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p02{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p03{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p04{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p05{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p06{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p07{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p08{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p09{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p10{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p11{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p12{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p13{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p14{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p15{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p16{01..45}██ | tr -d ' ')"
	    echo "$(eval $(echo printf '" %0.s"' {1..$base_column}))${x}$(echo $p17{01..45}██ | tr -d ' ')"

		### reset formatting.

	    echo ${x}

		unset b w l x z

		synthia-define_formatting

		### print bottom padding.

		yes '' | sed ${bottom_padding}q

		### add delay.

		sleep 2

		### clear.

		clear

	fi

}

aenigma-xmpp_intro() {

	dna-echo -m "First of all, a little introduction on how XMPP actually works"
	dna-echo -m "XMPP works very much like email. You can have a domain [amsterdamhacklab.xyz] and receive email for that domain on a server located at mx01.amsterdamhacklab.xyz, by using the appropriate DNS configuration, and also have other servers [webserver, mapserver, etc...] on other subdomains."
	dna-echo -m "In that case, a DNS MX record tells sending servers to direct mail intended for amsterdamhacklab.xyz to mx01.amsterdamhacklab.xyz."
	dna-echo -m "In the same way, the XMPP server for amsterdamhacklab.xyz can be located at ae01.amsterdamhacklab.xyz and a DNS SRV record tells any sending servers that XMPP for that domain [i.e. a message directed to mark@amsterdamhacklab.xyz] is handled by the server located at ae01.amsterdamhacklab.xyz."

	dna-press_to_continue

	dna-echo -m "$project_name fully embraces this opportunity."
	dna-echo -m "The domain for which you're setting up your new $project_name server can definitely be a domain connected to a bigger project, that already points to other servers, and for which XMPP is just another service."
	dna-echo -m "This will give you working @amsterdamhacklab.xyz JIDs [XMPP account addresses], but the $project_name server will reside at ae01.amsterdamhacklab.xyz, as in the following example:"
	dna-echo -m "${o-}Domain:${n-}               ${y-}amsterdamhacklab.xyz.${n-}"
	dna-echo -m "${o-}Website:${n-}              ${y-}amsterdamhacklab.xyz${n-} / ${y-}www.amsterdamhacklab.xyz${n-} [hosted by another server]."
	dna-echo -m "${o-}Your JID format:${n-}      ${y-}mark@amsterdamhacklab.xyz.${n-}"
	dna-echo -m "${o-}XMPP server:${n-}          ${y-}xmpp.amsterdamhacklab.xyz${n-}"

	dna-press_to_continue

	## add cluster info here.

	dna-echo -m "Now that you know how XMPP works, let's get your brand new $project_name server up and running!"

}

aenigma-set_secondary_node_hostname() {

	### set hostname according to the node_number from the newly received
	### config.

	echo "ae${node_number}.${domain}" > "/etc/hostname"

	hostname="$(cat /etc/hostname)"

	hostname -F "/etc/hostname"

}

aenigma-provision_nginx() {

	dna-get_hostname

	### stop sslh to avoid any conflicts with a brand new nginx installation.

	if systemctl is-active --quiet sslh
	then

		dna-echo_operation -h "stopping SSLH"

		service sslh stop

		dna-echo_operation -t

	fi

	### install nginx and configure ejabberd admin panel reverse proxy site.
	### here we use the distro provided nginx as the nginx.org repo does not
	### have nginx-common required for libnginx-mod-http-perl.

	dna-install_dependencies -p "nginx-common nginx libnginx-mod-http-perl"

	### increase names_hash_bucket_size to 128 in nginx config as the .onion
	### hostname is really long should it be later provisioned.

	dna-nginx_increase_names_hash_bucket_size_128

	### remove default nginx stuff.

	if [ -d "/etc/nginx/sites-enabled/" ]
	then
		rm -rf "/etc/nginx/sites-enabled"
	fi

	if [ -f "/etc/nginx/conf.d/default.conf" ]
	then
		rm "/etc/nginx/conf.d/default.conf"
	fi

	### install perl and upload module.
	### https://github.com/weiss/ngx_http_upload.

	mkdir -p "/usr/local/lib/perl"
	wget -O "/usr/local/lib/perl/upload.pm" "https://git.io/fNZgL"

	sed -i "s|my \$uri_prefix_components = 0;|my \$uri_prefix_components = 1;|g" "/usr/local/lib/perl/upload.pm"

	nginx_version="$(nginx -v)"
	echo

	dna-get_distro_name

	### if this is debian manually load perl module. ## vestigial

#	if [ "${distro_name}" = "debian" ]
#	then

#		if ! grep -q "ngx_http_perl_module.so" "/etc/nginx/nginx.conf"
#		then

#			sed -i "/^events {/i load_module modules\/ngx_http_perl_module.so;\n" "/etc/nginx/nginx.conf"

#		fi

#	fi

	if ! grep -q "/usr/local/lib/perl" "/etc/nginx/nginx.conf"
    then

        sed -i "/^http {$/a\ \n    perl_modules /usr/local/lib/perl;\n    perl_require upload.pm;" "/etc/nginx/nginx.conf"

    fi

	### create web locations and populate them.

	mkdir -p "/var/www/${project_name}/"
	chown -R "www-data:www-data" "/var/www/${project_name}/"

	mkdir -p "/var/www/upload"
	chown -R "www-data:www-data" "/var/www/upload/"

	cp "${source_dir}/conf/web/global/index.html" "/var/www/${project_name}/"

	### create and configure the $domain website.

	cp "${source_dir}/conf/nginx/domain" "/etc/nginx/conf.d/${domain}.conf"

	sed -i "s/hostname.xyz/${hostname}/g" "/etc/nginx/conf.d/${domain}.conf"
	sed -i "s/domain.xyz/${domain}/g" "/etc/nginx/conf.d/${domain}.conf"

	### delete default_server setting in $domain nginx conf file in case .onion
	### nginx conf is present and already has default_server setting itslef.

	if [ -f "/etc/nginx/conf.d/onion.conf" ]
	then

		if grep -q "default_server" "/etc/nginx/conf.d/onion.conf"
		then

			sed -i "s|443 default_server ssl|443 ssl|g" "/etc/nginx/conf.d/${domain}.conf"

		fi

	fi

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		### generate a random external_secret for the nginx ejabberd uploads.

		if [ -z "${nginx_ejabberd_uploads_external_secret-}" ]
		then
			dna-echo -m "Generating password for the postgresql/patroni postgres user"
			nginx_ejabberd_uploads_external_secret="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 24 | head -n 1)"
			dna-set_option -o "nginx_ejabberd_uploads_external_secret" -p "xmpp" -s "${nginx_ejabberd_uploads_external_secret}"
		fi

	fi

	sed -i "s|it-is-secret|${nginx_ejabberd_uploads_external_secret}|g" "/usr/local/lib/perl/upload.pm"

	### create and configure the websites for any and every $additional_domain.

	if [ ! "${additional_domains-null}" = "none" ]
	then

		additional_domains_array=(${additional_domains})
		for additional_domain in "${additional_domains_array[@]}"
	    do

			cp "${source_dir}/conf/nginx/additional_domain" "/etc/nginx/conf.d/${additional_domain}.conf"

			sed -i "s/additional_domain-var/${additional_domain}/g" "/etc/nginx/conf.d/${additional_domain}.conf"

			sed -i "s/hostname.xyz/${hostname}/g" "/etc/nginx/conf.d/${additional_domain}.conf"
			sed -i "s/domain.xyz/${domain}/g" "/etc/nginx/conf.d/${additional_domain}.conf"

	    done

	fi

	### start sslh back up if it's already installed.

	dna-echo_operation -h "starting SSLH back up"

	if which sslh &> /dev/null
	then

		service sslh start

	fi

	dna-echo_operation -t

	### start nginx back up.

	dna-echo_operation -h "starting NGINX back up"

	service nginx restart || true

	dna-echo_operation -t

}

aenigma-install_sslh() {

	dna-install_sslh

	### customize config file for aenigma

	dna-echo_operation -h "customizing SSLH config for ${project_name}"

	cp "${source_dir}/conf/sslh/etc-sslh.cfg" "/etc/sslh.cfg"

	dna-localize_sslh_install

	sed -i "s|domain-var|${domain}|g" "/etc/sslh.cfg"
	sed -i "s|hostname-var|${hostname}|g" "/etc/sslh.cfg"
	sed -i "s|ssh_port-var|${ssh_port}|g" "/etc/sslh.cfg"
	sed -i "s|xh.|xh${node_number}.|g" "/etc/sslh.cfg"

	if [ ! "${additional_domains-null}" = "none" ]
	then

		> "${tmp_project_dir}/additional_domains_sslh_xh.txt"
		additional_domains_array=(${additional_domains})
		for additional_domain in "${additional_domains_array[@]}"
	    do
			echo "    { name: \"tls\"; host: \"localhost\"; port: \"5443\"; sni_hostnames: [ \"xh${node_number}.${additional_domain}\"]; log_level: 1; }," >> "${tmp_project_dir}/additional_domains_sslh_xh.txt"
	    done

		> "${tmp_project_dir}/additional_domains_sslh_ae.txt"
		additional_domains_array=(${additional_domains})
		for additional_domain in "${additional_domains_array[@]}"
	    do
			echo "    { name: \"tls\"; host: \"localhost\"; port: \"5223\"; sni_hostnames: [ \"ae${node_number}.${additional_domain}\" ]; log_level: 1; }," >> "${tmp_project_dir}/additional_domains_sslh_ae.txt"
	    done

		sed -i "/\"xh/ r ${tmp_project_dir}/additional_domains_sslh_xh.txt" "/etc/sslh.cfg"
		sed -i "/\"ae/ r ${tmp_project_dir}/additional_domains_sslh_ae.txt" "/etc/sslh.cfg"

	fi

	service sslh restart

	dna-echo_operation -t

}

aenigma-restart_ejabberd_if_running() {

	### ########################################################################

	### restart ejabberd if it's already installed and running.

	### ########################################################################

	dna-get_hostname

	if dpkg -s ejabberd &> /dev/null
	then

		if systemctl is-active --quiet ejabberd
		then

			dna-echo_operation -h "restarting ejabberd on node: | ${l-}${hostname}${n-} |"

			if service ejabberd restart
			then

				dna-wait -s "16" -d "Now allowing ejabberd to start up correctly..."
				dna-echo_operation -t

			else

				echo
				dna-echo -e "ejabberd failed during restart"
				dna-echo -m "Continuing anyway as this should fix itself automatically by the end of the installation."
				sleep 3

			fi

			ejabberdctl status || true
			echo

		else

			dna-echo -s "ejabberd not running on node:${n-} | ${l-}${hostname}${n-} |."

		fi

	else

		dna-echo -m "ejabberd not installed on node: | ${l-}${hostname}${n-} |."
		dna-echo -m "not going to start its service."

	fi

}

aenigma-start_ejabberd() {

	### ########################################################################

	### start ejabberd if it's already installed.

	### ########################################################################

	dna-get_hostname

	if dpkg -s ejabberd &> /dev/null
	then

		dna-echo_operation -h "starting ejabberd on node: | ${l-}${hostname}${n-} |"

		if service ejabberd start
		then

			dna-wait -s "16" -d "Now allowing ejabberd to start up correctly..."
			dna-echo_operation -t

		else

			echo
			dna-echo -e "ejabberd failed during restart"
			dna-echo -m "Continuing anyway as this should fix itself automatically by the end of the installation."
			sleep 3

		fi

		ejabberdctl status || true
		echo

	else

		dna-echo -m "ejabberd not installed on node: | ${l-}${hostname}${n-} |."
		dna-echo -m "not going to start its service."

	fi

}

aenigma-stop_ejabberd() {

	### ########################################################################

	### stop ejabberd if it's already installed and running.

	### ########################################################################

	dna-get_hostname

	if dpkg -s ejabberd &> /dev/null
	then

		if systemctl is-active --quiet ejabberd
		then

			dna-echo_operation -h "stopping ejabberd on node: | ${l-}${hostname}${n-} |"

			service ejabberd stop &> /dev/null || true

			dna-wait -s "4" -d "Now allowing ejabberd to shut down correctly..."

			dna-echo_operation -t

		else

			dna-echo -s "ejabberd already stopped on node:${n-} | ${l-}${hostname}${n-} |."

			service ejabberd stop &> /dev/null || true

		fi

	else

		dna-echo -m "ejabberd not installed on node: | ${l-}${hostname}${n-} |."
		dna-echo -m "no need to stop its service"

	fi

}

aenigma-start_patroni() {

	### ########################################################################

	### start patroni if it's already installed.

	### ########################################################################

	dna-get_hostname

	if dpkg -s python-pip &> /dev/null && pip list 2> /dev/null | grep -q patroni && [ -f "/etc/systemd/system/patroni.service" ] && [ -d "/usr/lib/postgresql/" ]
	then

		dna-echo_operation -h "starting patroni on node: | ${l-}${hostname}${n-} |"

		service patroni start

		dna-wait -s "16" -d "Now allowing patroni to start up correctly..."

		dna-echo_operation -t

	else

		dna-echo -m "patroni not [fully] installed on node: | ${l-}${hostname}${n-} |."
		dna-echo -m "not going to start its service."

	fi

}

aenigma-stop_patroni() {

	### ########################################################################

	### stop patroni if it's already installed and running.

	### ########################################################################

	dna-get_hostname

	if dpkg -s python-pip &> /dev/null && pip list 2> /dev/null | grep -q patroni && [ -f "/etc/systemd/system/patroni.service" ] && [ -d "/usr/lib/postgresql/" ]
	then

		if systemctl is-active --quiet patroni
		then

			dna-echo_operation -h "stopping patroni on node: | ${l-}${hostname}${n-} |"

			service patroni stop &> /dev/null || true

			dna-wait -s "16" -d "Now allowing patroni to shut down correctly..."

			dna-echo_operation -t

		else

			dna-echo -s "patroni already stopped on node:${n-} | ${l-}${hostname}${n-} |."

			service patroni stop &> /dev/null || true

		fi

	else

		dna-echo -m "patroni not installed on node: | ${l-}${hostname}${n-} |."
		dna-echo -m "no need to stop its service"

	fi

}

aenigma-cluster_operations_preflight() {

	#### ensure cluster is enabled and this is the primary node.

	if [ "${cluster_enabled}" = "single" ]
	then
		dna-echo -e "This script is only intended to be executed when running ${project_name} in a cluster of several nodes."
		dna-exit
	fi

	if [ "${cluster_mode-null}" = "secondary" ]
	then
		dna-echo -e "This script must only be run on the primary node of your ${project_name} instance."
		dna-exit
	fi

	#### ensure the nodes_db file exists.

	if [ ! -f "${nodes_db_ipv4_path}" ]
	then
		dna-echo -e " | ${n-}nodes_db${r-} | file not found."
		dna-echo -m "Either the initial installation was never completed or the file was somehow removed."
		dna-echo -m "Re-run the installation to fix this."
		dna-exit
	fi

	#### ensure ip info is current.

	aenigma-check_ip_info_current

}

aenigma-clusterize() {

	### ########################################################################

	### remotely provision secondary nodes to form an ejabberd cluster.

	### ########################################################################

	### evaluate arguments.

	### ########################################################################

	while getopts ":e" arguments
	do
	    case $arguments in
	        e)	clusterize_force_reinstall_etcd_cluster="y"
	            ;;
	        \?)	dna-echo -e "Invalid option | ${n-}-${OPTARG}${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
	            dna-exit
	            ;;
	    esac
	done
	OPTIND=1

	### ########################################################################

	### check / enable clusterization.

	### ########################################################################

	if [ ! "${cluster_enabled-null}" = "cluster" ]
	then
		dna-echo -e "${project_name} was not installed in cluster mode."
		dna-ask_for_boolean -d "y" -q "Do you wish to turn this instance into a cluster one?"
		if [ "${boolean_output}" = "y" ]
		then
			dna-echo -m "Ok, setting clusterization to enabled."
			dna-set_option -o cluster_enabled -p cluster -s "cluster"
		else
			dna-echo -m "Ok, leaving clusterization off for now."
			dna-exit
		fi
	fi

	### ########################################################################

	### ensure cluster is enabled and this is the primary node.

	### ###########################################################################

	if [ -z "${cluster_mode-}" ]
	then
		dna-echo -m "This script must only be run on the primary node of ${project_name}."
		dna-ask_for_boolean -d "n" -q "Do you confirm that this will in fact be the primary node for your ${project_name} instance?"
		if [ "${boolean_output}" = "y" ]
		then
			dna-echo -m "Ok, proceeding..."
			dna-set_option -o cluster_mode -p cluster -s "primary"
		else
			dna-echo -m "Ok, no problem. If this is a secondary node all you need to do is run the standard ${project_name} installation and select | secondary | when asked about the node type during clusterization setup."
			dna-exit
		fi
	elif [ "${cluster_mode}" = "secondary" ]
	then
		dna-echo -e "This script must only be run on the primary node of ${project_name}."
		dna-echo -m "You've previously specified during install that this will instead be a secondary node."
		dna-echo -m "If that's the case then all you need to do is run the standard ${project_name} installation and select | secondary | when asked about the node type during clusterization setup."
		dna-echo -m "Otherwise if you actually intend to use this machine as your primary node, simply re-run the standard installation and specify so when promted."
	else
		dna-echo -s "${project_name} | ${n-}clusterization mode${g-} | already set to: | ${n-}primary${g-} |."
	fi

	### ########################################################################

	### ensure installation has started on primary node.

	### ########################################################################

	dna-ask_for_boolean -d "y" -q "Have you already started the ${project_name} installation on your secondary node?"
	if [ "${boolean_output}" = "y" ]
	then
		dna-echo -m "Ok, continuing..."
	else
		dna-echo -m "Ok, no problem. Start the installation [or re-installation] on your secondary node now from scratch just by running the standard: | ${o-}bash ${project_name}/setup${n-} | command."
		dna-echo -m "When prompted to start this very script come back here and continue on."
		dna-press_to_continue
	fi

	### ########################################################################

	### ensure the nodes_db files are up to date with the IP address[es] of this
	### machine.

	### ########################################################################

	aenigma-initialize_nodes_db_files

	### ########################################################################

	### create a nodes_db_initial file so that we can absolutely detect if any
	### nodes were actually added during this script [rather than simply
	### re-provisioned or skipped].

	### ########################################################################

	aenigma-create_nodes_db_initial

	validate_input() {

		dna-validate_ipv4 -s -i "${input}"

		if [ "${ip_is_valid}" = "y" ]
		then
			valid="y"
		else
			dna-echo -e "The value you specified does NOT appear to be a valid IPv4 address."
			valid="n"
		fi

	    unset silent

	}

	dna-set_value -d "Now enter the public IPv4 address displayed on your secondary node:" -v "y"

	secondary_node_ipv4="${value}"

	### ########################################################################

	### now add the secondary node's IPv4 to nodes_db_ipv4.

	### ########################################################################

	if grep -q "^${secondary_node_ipv4}$" "${nodes_db_ipv4_path}"
	then

		#### specified secondary node is already registered.

		secondary_node_number="$(grep -n "${secondary_node_ipv4}" "${nodes_db_ipv4_path}" | grep -Eo '^[^:]+' | xargs printf %02d)"
		dna-echo -m "The server with IPv4 | ${y-}${secondary_node_ipv4}${n-} | is already registered as part of this cluster with node number | ${o-}${secondary_node_number}${n-} |."
		dna-ask_for_boolean -d "y" -q "Would you like to re-provision it?"

		if [ "${boolean_output}" = "y" ]
		then
			dna-echo -m "Ok, re-provisioning node number | ${o-}${secondary_node_number}${n-} |..."
		else
			dna-echo -m "Ok, skipping re-provisioning for node | ${o-}${secondary_node_number}${n-} |."
			dna-exit
		fi

	else

		#### specified secondary node is new to this cluster.

		echo "${secondary_node_ipv4}" >> "${nodes_db_ipv4_path}"
		secondary_node_number="$(grep -n "${secondary_node_ipv4}" "${nodes_db_ipv4_path}" | grep -Eo '^[^:]+' | xargs printf %02d)"
		dna-echo -m "Ok, added server with IPv4 | ${y-}${secondary_node_ipv4}${n-} | to this cluster with node number | ${o-}${secondary_node_number}${n-} |."

	fi

	aenigma-create_nodes_db_numbered

	### ########################################################################

	### generate or read and print the clusterization SSH pubkey and prompt the
	### user to configure it on the seondary node.

	### ########################################################################

	if [ -f "/root/.ssh/aenigma_cluster.pub" ]
	then
		aenigma_cluster_ssh_keypair_preexisting="y"
	else
		dna-generate_ssh_keypair -n "aenigma_cluster" -h "ae01.${domain}"
	fi

	ssh_keypair_pubkey_path="/root/.ssh/aenigma_cluster.pub"

	if [ "${aenigma_cluster_ssh_keypair_preexisting-null}" = "y" ]
	then

		#### the aenigma_cluster SSH keypair was already generated during a
	    #### previous clusterization. check to see if a connection to the
	    #### secondary nodes already succeeds [i.e. because of a previous
		#### provisioning of such node].

		if ssh -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${secondary_node_ipv4}" exit  &> /dev/null
		then
			ssh_connection_to_secondary_node_succeded="y"
		fi

	fi

	if [ "${ssh_connection_to_secondary_node_succeded-null}" = "y" ]
	then

		#### ssh back into the secondary node, and inform it that we are already
	    #### able to connect to it.

		ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${secondary_node_ipv4}" "exec bash -li" > /dev/null <<END
set -eu
touch "/tmp/aenigma/aenigma-cluster-ssh-connection-successful"
exit
END
		echo

		dna-echo -s "SSH connection to secondary node succeeded. Now continuing installation..."

	else

		#### the ssh pubkey must be specified by the user on the secondary node.

		dna-echo -m "Now, when prompted, enter the following SSH public key on the secondary node:"
		dna-echo -m " | ${y-}$(cat ${ssh_keypair_pubkey_path})${n-} | "

		check_condition() {

			if ssh -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${secondary_node_ipv4}" exit &> /dev/null
			then
				condition_verified=y
			fi

		}

		dna-wait_for -i "8" -d "Now waiting for successful connection to the secondary node..." -m "64" -e "Timeout exceed for SSH connection test to secondary node."

		if [ "${wait_output}"  = "ok" ]
		then
			dna-echo -s "SSH connection to secondary node succeeded. Now continuing installation..."
		else
			dna-echo -e "SSH connection to secondary node failed."
			dna-echo -m "Please ensure the secondary node's IPv4 address is actually | ${y}${secondary_node_ipv4}${n-} |, that its SSH port is | ${y}42022${n-} |, and that its firewall isn't blocking connections on that port."
			dna-exit
		fi

		unset wait_output
		sleep 1

	fi

	### ########################################################################

	### fetch and prompt user to confirm the clusterization token visualised on
	### the secondary node to ensure we're speaking to the right machine.

	### ########################################################################

	if secondary_node_token="$(ssh -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${secondary_node_ipv4}" cat /tmp/aenigma/aenigma-cluster-token)"
	then
		dna-echo -s "Token on secondary node found."
		dna-echo -m "Its value is: | ${y-}${secondary_node_token}${n-} |."
		dna-ask_for_boolean -d "y" -q "Does this token match what you see on the secondary node?"
		if [ "${boolean_output}" = "y" ]
		then
			dna-echo -m "Ok, continuing..."
		else
			dna-echo -m "No problem. Please ensure the secondary node in question's IPv4 address is actually | ${y}${secondary_node_ipv4}${n-} | and try again."
			dna-exit
		fi
	else
		dna-echo -e "Token on secondary node not found."
		dna-echo -m "Please ensure you've followed the configuration steps on your secondary node up to the point where it displays the clusterization token and try again."
		dna-exit
	fi

	### ########################################################################

	### add blank line to nodes_db_ipv6 if it doesn't yet exist for sed matching
	### below.

	### ########################################################################

	nodes_db_ipv6_lines="$(wc -l < ${nodes_db_ipv6_path})"
	if [ "${nodes_db_ipv6_lines}" -lt "${secondary_node_number}" ]
	then
		echo " " >> "${nodes_db_ipv6_path}"
	fi

	### ########################################################################

	### if token is an ipv6 address then import it here and add it to nodes_db_ipv6
	### otherwise specify lack thereof.

	### ########################################################################

	echo "${secondary_node_token}" > "${tmp_project_dir}/secondary_node_token"

	if grep -q ":" "${tmp_project_dir}/secondary_node_token"
	then

		#### the secondary node is IPv6 enabled, add this to nodes_db_ipv6.

		secondary_node_ipv6="$(cat ${tmp_project_dir}/secondary_node_token)"
		sed -i "${secondary_node_number}s/.*/${secondary_node_ipv6}/" "${nodes_db_ipv6_path}"

	else

		#### the secondary node is NOT IPv6 enabled, specify this in
		#### nodes_db_ipv6.

		sed -i "${secondary_node_number}s/.*/null/" "${nodes_db_ipv6_path}"

	fi

	### ########################################################################

	### update routing info on this [the primary] node.

	### ########################################################################

	aenigma-update_routing_info

	### ########################################################################

	### push and update routing info on all secondary nodes [both existing and
	### the new one we're now provisioning] to allow them to connect to each
	### other correctly.

	### ########################################################################

	aenigma-push_routing_info

	### ########################################################################

	### perform updated DNS checks based on the newly registered information
	### about the secondary node [only if new nodes were actually added].

	### ########################################################################

	aenigma-compare_nodes_db_initial

	if [ "${nodes_db_initial_diff}" = "diff" ]
	then

	    aenigma-perform_dns_checks

	fi

	### ########################################################################

	### push node_number to the secondary node.

	### ########################################################################

	echo "${secondary_node_number}" > "/${tmp_project_dir}/aenigma_cluster_secondary_node_number"
	/usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "/${tmp_project_dir}/aenigma_cluster_secondary_node_number" "root@${secondary_node_ipv4}:${conf_cluster_dir}/node_number"

	### ########################################################################

	### push config to the secondary node.

	### ########################################################################

	/usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" -r "${conf_backup_dir}" "root@${secondary_node_ipv4}:${conf_dir}/"
	/usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "${conf_dir}/xmpp/domain" "root@${secondary_node_ipv4}:${conf_dir}/xmpp/"
	/usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "${conf_dir}/xmpp/additional_domains" "root@${secondary_node_ipv4}:${conf_dir}/xmpp/"
	/usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "${conf_dir}/xmpp/nginx_ejabberd_uploads_external_secret" "root@${secondary_node_ipv4}:${conf_dir}/xmpp/"
	/usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "/opt/ejabberd/.erlang.cookie" "root@${secondary_node_ipv4}:/opt/ejabberd/"

	### ########################################################################

	### push postgresql/patroni passwords to the secondary node.

	### ########################################################################

	/usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" -r "${conf_dir}/postgresql/" "root@${secondary_node_ipv4}:${conf_dir}/"

	### ########################################################################

	### push wildcard LE TLS cert for $domain to the secondary node.

	### ########################################################################

	/usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" -r "${tls_project_dir}" "root@${secondary_node_ipv4}:/etc/ssl/"

	echo

	### ########################################################################

	### apply and source new hostname on newly provisioned node so that etcd
	### will start up correctly right from the start.

	### ########################################################################

	#### aenigma-exec block

	#### #######################################################################

	## >>> define the commands that will be passed to the aenigma-exec script
	## >>> on the remote machine to be run, one per line between the quotes.

	aenigma-exec() {

	    bash openspace42/aenigma/tools/aenigma-exec -c "
aenigma-set_secondary_node_hostname
"

	}

	## >>>> ssh into the secondary node, declare the aenigma-exec
	## >>>> function, and run it inside of the subshell.

	ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${secondary_node_ipv4}" "$(declare -f aenigma-exec); export -f aenigma-exec; exec bash -li" <<END
set -eu
. /root/openspace42/aenigma/functions
synthia-source_dna
synthia-define_vars
dna-read_conf_settings
aenigma-exec
exit
END

	echo

	#### #######################################################################

	### ########################################################################

	### install etcd cluster if new nodes were added or if force argument
	### provided.

	### ########################################################################

	if [ "${nodes_db_initial_diff}" = "diff" ] || [ -n "${clusterize_force_reinstall_etcd_cluster-}" ]
	then

	    #### re-run the etcd cluster installation now that we have a new node.

	    aenigma-install_etcd_cluster

	fi

	### ########################################################################

	### apply new configs and restart patroni, haproxy, and ejabberd on existing
	### nodes [not the one we're provisioning now, which we exclude].

	### ########################################################################

	aenigma-apply_and_restart_patroni_haproxy_ejabberd -e "${secondary_node_ipv4}"

	### ########################################################################

	### ssh into the secondary node, and inform it that the setup here on the
	### primary node has been completed.

	### ########################################################################

	ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${secondary_node_ipv4}" "exec bash -li" > /dev/null <<END
set -eu
touch "/tmp/aenigma/aenigma-cluster-primary-node-setup-complete"
exit
END
	echo

	### ########################################################################

	### prompt the user to finish the installation on secondary node by
	### following the next steps on the other node.

	### ########################################################################

	dna-echo -s "Primary_node-level configuration complete."
	dna-echo -m "You can now return to your secondary node and finish running the installation from there."

	### ########################################################################

	### wait for the installation on the secondary node to be complete.

	### ########################################################################

	check_condition() {

	    if ssh -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${secondary_node_ipv4}" "rm /tmp/aenigma_secondary_node_provisioning_complete" &> /dev/null
	    then
	        condition_verified=y
	    fi

	}

	dna-wait_for -i "8" -d "Now waiting for setup on the secondary node to be complete..."
	dna-echo -s "Secondary node setup complete. Clusterization successful."

	unset wait_output
	sleep 1

	### ########################################################################

	### remove tmp_project_dir.

	### ########################################################################

	dna-remove_tmp_project_dir

	### ########################################################################

}

aenigma-stop_ejabberd_patroni_clusterwide() {

	### ########################################################################

	### stop ejabberd and subsequently patroni on all nodes of the cluster
	### if they're already installed and running.

	### ########################################################################

	### ejabberd.

	### ########################################################################

	#### stop ejabberd locally.

	aenigma-stop_ejabberd

	#### stop ejabberd on secondary nodes.

	##### aenigma-exec block

	##### ######################################################################

	## >>> get online nodes.

	aenigma-get_nodes_online

	## >>> define the commands that will be passed to the aenigma-exec script
	## >>> on the remote machine to be run, one per line between the quotes.

	aenigma-exec() {

		bash openspace42/aenigma/tools/aenigma-exec -c "
aenigma-stop_ejabberd
"

	}

	## >>> connect to every online node and perform operations remotely.

	for online_node_ipv4 in $(cat < "${nodes_db_online_path}")
	do

		## >>>> ensure we don't try connecting to ourselves [therefore ignore
		## >>>> the first line of nodes_db_ipv4].

		if [ ! "${online_node_ipv4}" = "${public_ipv4}" ]
		then

			## >>>> ssh into the secondary node, declare the aenigma-exec
			## >>>> function, and run it inside of the subshell.

			ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${online_node_ipv4}" "$(declare -f aenigma-exec); export -f aenigma-exec; exec bash -li" <<END
set -eu
. /root/openspace42/aenigma/functions
synthia-source_dna
synthia-define_vars
dna-read_conf_settings
aenigma-exec
exit
END

		fi

		echo

	done

	unset online_node_ipv4

	##### ######################################################################

	### ########################################################################

	### patroni.

	### ########################################################################

	#### stop patroni locally.

	aenigma-stop_patroni

	#### stop patroni on secondary nodes.

	##### aenigma-exec block

	##### ######################################################################

	## >>> get online nodes.

	aenigma-get_nodes_online

	## >>> define the commands that will be passed to the aenigma-exec script
	## >>> on the remote machine to be run, one per line between the quotes.

	aenigma-exec() {

		bash openspace42/aenigma/tools/aenigma-exec -c "
aenigma-stop_patroni
"

	}

	## >>> connect to every online node and perform operations remotely.

	for online_node_ipv4 in $(cat < "${nodes_db_online_path}")
	do

		## >>>> ensure we don't try connecting to ourselves [therefore ignore
		## >>>> the first line of nodes_db_ipv4].

		if [ ! "${online_node_ipv4}" = "${public_ipv4}" ]
		then

			## >>>> ssh into the secondary node, declare the aenigma-exec
			## >>>> function, and run it inside of the subshell.

			ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${online_node_ipv4}" "$(declare -f aenigma-exec); export -f aenigma-exec; exec bash -li" <<END
set -eu
. /root/openspace42/aenigma/functions
synthia-source_dna
synthia-define_vars
dna-read_conf_settings
aenigma-exec
exit
END

		fi

		echo

	done

	unset online_node_ipv4

	##### ######################################################################

	### ########################################################################

}

aenigma-add_ejabberd_repo() {

	### install dependencies.

	dna-install_wget

	### add repo.

	dna-check_dpkg_availability
	dna-install_dependencies -p "apt-transport-https"

	### add the apt.jabber.at repo.

	dna-get_distro_codename

	echo "deb https://apt.jabber.at ${distro_codename} ejabberd" > "/etc/apt/sources.list.d/jabber.at.list"

	### pin this repo at max priority in Debian in case testing/unstable repos
	### are enabled.

	dna-get_distro_name

	if [ "${distro_name}" = "debian" ]
	then
		touch "/etc/apt/preferences"
		if ! grep -q "apt.jabber.at" "/etc/apt/preferences"
		then
			cat <<EOF >> "/etc/apt/preferences"

Package: *
Pin: origin "apt.jabber.at"
Pin-Priority: 1000

EOF
		fi
	fi

	### add key.

	if ! apt-key list 2> /dev/null | grep "jabber.at" &> /dev/null
	then
		wget -qO- https://apt.jabber.at/gpg-key | apt-key add -
		apt-get update
	fi

}

aenigma-get_ejabberd_installed_version() {

	if dpkg -s ejabberd &> /dev/null
	then
		ejabberd_installed_version="$(dpkg -s ejabberd | grep '^Version:' | sed "s|Version: ||" | cut -d ':' -f 2 | cut -f1 -d'-')"
	else
		ejabberd_installed_version=none
	fi

}

aenigma-install_ejabberd_config() {

	dna-echo -m "Now creating ejabberd directory and setting custom aenigma config..."

	source_ejab_conf_file="${source_dir}/conf/ejabberd/ejabberd-${hardcoded_package_version_ejabberd}.yml"

	if [ ! -f "${source_ejab_conf_file}" ]
	then

		### fallback to old ejabberd.yml in case the ejabberd candidate install
		### version isn't among the ones we have config files for.

		source_ejab_conf_file="${source_dir}/conf/ejabberd/ejabberd-${hardcoded_package_version_ejabberd_fallback}.yml"

	fi

	cp "${source_ejab_conf_file}" "/opt/ejabberd/conf/ejabberd.yml"

	### install captcha script.

	dna-install_dependencies -p "imagemagick"

	if [ -f "/opt/ejabberd/captcha.sh" ]
	then
		rm "/opt/ejabberd/captcha.sh"
	fi

	curl "https://raw.githubusercontent.com/processone/ejabberd/master/tools/captcha.sh" > "/opt/ejabberd/captcha.sh"
	echo

	### set permissions.

	aenigma-set_permissions

	dna-echo -s "Finished creating ejabberd directory and setting custom aenigma config."

}

aenigma-set_permissions() {

	### ########################################################################

	### set permissions globally.

	### ########################################################################

	dna-echo_operation -h "setting permissions"

	### create the aenigma group if it doesn't exist.

	if ! grep -q "aenigma" "/etc/group"
	then
		addgroup aenigma
		echo
	fi

	### ~/.ssh/.

	if [ -d "/root/.ssh/" ]
	then
		chmod 755 "/root/.ssh/"
	fi

	if [ -f "/root/.ssh/authorized_keys" ]
	then
		chown "root:root" "/root/.ssh/authorized_keys"
		chmod 600 "/root/.ssh/authorized_keys"
	fi

	### ejabberd.

	### ########################################################################

	#### /opt/ejabberd/.

	if id "ejabberd" &> /dev/null && [ -d "/opt/ejabberd/" ]
	then
		chown -R "ejabberd:ejabberd" "/opt/ejabberd/"
	fi

	#### /opt/ejabberd/.erlang.cookie.

	if id "ejabberd" &> /dev/null && [ -f "/opt/ejabberd/.erlang.cookie" ]
	then
		chown "ejabberd:ejabberd" "/opt/ejabberd/.erlang.cookie"
	fi

	#### /opt/ejabberd/captcha.sh.

	if [ -f "/opt/ejabberd/captcha.sh" ]
	then

		chmod +x "/opt/ejabberd/captcha.sh"

	fi

	### ########################################################################

	### $tls_project_dir.
	### assign new permissions to tls_project_dir so that postgresql and etcd
	### too can read from it, if ejabberd doesn't exist yet at least set group.

	if id "ejabberd" >/dev/null 2>&1
	then
		chown -R "ejabberd:aenigma" "${tls_project_dir}"
	else
		chown -R "root:aenigma" "${tls_project_dir}"
	fi

	chmod -R 770 "${tls_project_dir}"

	### /data/.

	mkdir -p "/data/"

	if id "postgres" &>/dev/null
	then
		chown -R postgres:postgres "/data/"
	fi

	chmod -R 700 "/data/"

	dna-echo_operation -t

}

aenigma-create_ejabberd_locations() {

	### create ejabberd directory for correct .erlang.cookie placement on
	### secondary nodes before ejabberd is installed from deb package.

	dna-echo_operation -h "creating /opt/ejabberd/ directory"

	mkdir -p "/opt/ejabberd/"

	dna-echo_operation -t

	### set permissions.

	aenigma-set_permissions

}

aenigma-verify_ejabberd_deb_package() {

	### ########################################################################

	### https://www.process-one.net/index.php/docs/verifying_process_one_downloads_integrity/

	### ########################################################################

	### evaluate arguments.

	### ########################################################################

	unset check_silently

	while getopts ":s" arguments
    do
        case "${arguments}" in
            s)	check_silently="y"
                ;;
            \?)	dna-echo -e "Invalid option | ${n-}-$OPTARG${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
                dna-exit
                ;;
        esac
    done
    OPTIND=1

	### ########################################################################

	dna-install_dependencies -p "gnupg"

	gpg --import "${source_dir}/conf/gpg/process-one.asc"
	echo

	wget -qO "${tmp_project_dir}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb.sum" "https://www.process-one.net/downloads/downloads-action.php?file=/ejabberd/${hardcoded_package_version_ejabberd}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb.sum"
	wget -qO "${tmp_project_dir}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb.asc" "https://www.process-one.net/downloads/downloads-action.php?file=/ejabberd/${hardcoded_package_version_ejabberd}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb.asc"

	### sha256.

	cd "${tmp_project_dir}"

	if sha256sum -c "${tmp_project_dir}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb.sum"
	then

		if [ -z "${check_silently-}" ]
		then

			echo
			dna-echo -s "ejabberd SHA256 file integrity check succeeded."

		fi

	else

		if [ -z "${check_silently-}" ]
		then

			echo
			dna-echo -e "ejabberd SHA256 file integrity check failed."

		fi

		return 1

	fi

	if [ -n "${check_silently-}" ]
	then
		echo
	fi

	### gpg.

	if gpg --verify "${tmp_project_dir}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb.asc"
	then

		if [ -z "${check_silently-}" ]
		then

			echo
			dna-echo -s "ejabberd GPG file integrity check succeeded."

		fi

	else

		if [ -z "${check_silently-}" ]
		then

			echo
			dna-echo -e "ejabberd GPG file integrity check failed."

		fi

		return 1

	fi

	if [ -n "${check_silently-}" ]
	then
		echo
	fi

}

aenigma-install_ejabberd_deb() {

	### ########################################################################

	### https://computingforgeeks.com/how-to-install-ejabberd-xmpp-server-on-ubuntu-18-04-ubuntu-16-04/

	### ########################################################################

	dna-echo_operation -h "installing ejabberd"

	### ensure dpkg is available before proceeding.

	dna-check_dpkg_availability

	### download .deb package.

	if [ -f "${tmp_project_dir}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb" ]
	then

		if aenigma-verify_ejabberd_deb_package -s
		then

			dna-echo -s "ejabberd deb file already downloaded and integer."

		else

			echo
			dna-echo -e "ejabberd deb file already downloaded but NOT integer."
			dna-echo -m "Downloading it once again now..."

			ejabberd_deb_file_download_needed="y"

		fi

	else

		ejabberd_deb_file_download_needed="y"

	fi

	if [ "${ejabberd_deb_file_download_needed-null}" = "y" ]
	then

		### download .deb package file.

		wget -O "${tmp_project_dir}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb" "https://www.process-one.net/downloads/downloads-action.php?file=/ejabberd/${hardcoded_package_version_ejabberd}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb"

		### verify .deb package file.

		if ! aenigma-verify_ejabberd_deb_package
		then

			dna-ask_for_boolean -d "n" -q "Would you like to continue anyway [strongly NOT recommended]?"

			if [ "${boolean_output}" = "y" ]
			then

				dna-echo -m "Ok, continuing anyway..."

			else

				dna-echo -m "Ok, aborting..."
				dna-exit

			fi

		fi

	fi

	### install .deb package.

	dpkg -i "${tmp_project_dir}/ejabberd_${hardcoded_package_version_ejabberd_extended}_amd64.deb" || true

	echo

	### link binaries.

	cp "/opt/ejabberd-${hardcoded_package_version_ejabberd}/bin/ejabberdctl" "/usr/sbin/"
	sed -i "s|^ROOT_DIR.*|ROOT_DIR=\"/opt/ejabberd-${hardcoded_package_version_ejabberd}/\"|g" "/usr/sbin/ejabberdctl"

	dna-echo_operation -t

	unset ejabberd_deb_file_download_needed

	############################################################################

}

aenigma-localize_ejabberd_install() {

	### evaluate arguments.

	### ########################################################################

	while getopts ":s" arguments
	do
        case $arguments in
            s)	skip_ejabberd_stop_start_operations="y"
	            ;;
            \?)	dna-echo -e "Invalid option | ${n-}-${OPTARG}${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
	            dna-exit
	            ;;
        esac
    done
    OPTIND=1

	### ########################################################################

	dna-get_hostname

	### stop ejabberd if it's already installed and running.

	if [ -z "${skip_ejabberd_stop_start_operations-}" ]
	then

		aenigma-stop_ejabberd

	fi

	### perform customizations on the ejabberd config files.

	### ########################################################################

	#### ejabberd.yml.

	if [ ! "${additional_domains-null}" = "none" ]
	then

		> "${tmp_project_dir}/additional_domains_ejabberd.txt"
		additional_domains_array=(${additional_domains})
		for additional_domain in "${additional_domains_array[@]}"
	    do
			echo "  - \"${additional_domain}\"" >> "${tmp_project_dir}/additional_domains_ejabberd.txt"
		done

		sed -i "/  - \"domain.xyz\"/ r ${tmp_project_dir}/additional_domains_ejabberd.txt" "/opt/ejabberd/conf/ejabberd.yml"

	fi

	sed -i "s/domain.xyz/${domain}/g" "/opt/ejabberd/conf/ejabberd.yml"
	sed -i "s/hostname.xyz/${hostname}/g" "/opt/ejabberd/conf/ejabberd.yml"

	sed -i "s/public_ipv4-var/${public_ipv4}/g" "/opt/ejabberd/conf/ejabberd.yml"

	sed -i "s/admin_mail/${admin_mail}/g" "/opt/ejabberd/conf/ejabberd.yml"

	sed -i "s/ae.@/ae${node_number}.@/g" "/opt/ejabberd/conf/ejabberd.yml"

	sed -i "s/xh.@/xh${node_number}.@/g" "/opt/ejabberd/conf/ejabberd.yml"
	sed -i "s/xm.@/xm${node_number}.@/g" "/opt/ejabberd/conf/ejabberd.yml"
	sed -i "s/xp.@/xp${node_number}.@/g" "/opt/ejabberd/conf/ejabberd.yml"
	sed -i "s/xr.@/xr${node_number}.@/g" "/opt/ejabberd/conf/ejabberd.yml"
	sed -i "s/xv.@/xv${node_number}.@/g" "/opt/ejabberd/conf/ejabberd.yml"

	sed -i "s/nginx_ejabberd_uploads_external_secret-var/${nginx_ejabberd_uploads_external_secret}/g" "/opt/ejabberd/conf/ejabberd.yml"

	sed -i "s/ejabberd-psql-user-password-var/${patroni_ejabberd_password}/g" "/opt/ejabberd/conf/ejabberd.yml"

	#### ejabberdctl.cfg.

	sed -i "/INET_DIST_INTERFACE=/c\INET_DIST_INTERFACE=${public_ipv4}" "/opt/ejabberd/conf/ejabberdctl.cfg"

	sed -i "/# ERLANG_NODE/d" "/opt/ejabberd/conf/ejabberdctl.cfg"
	sed -i "/ERLANG_NODE=/c\ERLANG_NODE=ejabberd@${hostname}" "/opt/ejabberd/conf/ejabberdctl.cfg"

	### ########################################################################

	### set permissions now that user ejabberd exists while ejabberd has still
	### not started yet.

	aenigma-set_permissions

	if [ -z "${skip_ejabberd_stop_start_operations-}" ]
	then

		### finally start ejabberd.

		aenigma-start_ejabberd

	fi

	unset skip_ejabberd_stop_start_operations

	############################################################################
	############################################################################

}

aenigma-enable_ejabberd() {

	############################################################################

	dna-echo_operation -h "enabling ejabberd"

	### install service.

	dna-install_dependencies -p "mlocate"

	updatedb

	cp "/opt/ejabberd-${hardcoded_package_version_ejabberd}/bin/ejabberd.service" "/etc/systemd/system"

	### reduce timeout to 60 seconds to avoid waiting for a failed ejabberd
	### startup for no reason.

	sed -i "s|TimeoutSec=300|TimeoutSec=60|g" "/etc/systemd/system/ejabberd.service"

	### enable service.

	systemctl daemon-reload
	systemctl enable --now ejabberd || true
	echo

	dna-echo_operation -t

	############################################################################

}

aenigma-create_domains_db() {

	if [ -f "${domains_db_path}" ]
	then
		rm "${domains_db_path}"
		touch "${domains_db_path}"
	fi

	echo "${domain}" > "${domains_db_path}"

	if [ ! "${additional_domains-null}" = "none" ]
	then

		additional_domains_array=(${additional_domains})
		for additional_domain in "${additional_domains_array[@]}"
	    do
			echo "${additional_domain}" >> "${domains_db_path}"
		done

	fi

}

aenigma-create_nodes_db_numbered() {

	### create a node-numbered version of "nodes_db".

	if [ -f "${nodes_db_numbered_path}" ]
	then
		rm "${nodes_db_numbered_path}"
		touch "${nodes_db_numbered_path}"
	fi

	array_iter=1
	IFS=$'\n'
	set -f

	for node_ipv4 in $(cat < "${nodes_db_ipv4_path}")
	do
		array_iter_padded="$(echo ${array_iter} | xargs printf %02d)"
		echo -e "${node_ipv4}\tae${array_iter_padded}.${domain}" >> "${nodes_db_numbered_path}"
		array_iter=$((array_iter + 1))
	done

	unset node_ipv4
	IFS=$' \t\n'
	set +f

}

aenigma-create_nodes_db_initial() {

	cp "${nodes_db_numbered_path}" "${nodes_db_initial_path}"

}

aenigma-compare_nodes_db_initial() {

	if diff "${nodes_db_numbered_path}" "${nodes_db_initial_path}" > "/dev/null"
	then
	    export nodes_db_initial_diff="same"
	else
	    export nodes_db_initial_diff="diff"
	fi

}

aenigma-initialize_nodes_db_files() {

	mkdir -p "${conf_cluster_nodes_dir}"

	dna-get_public_ipv4 -s

	dna-check_ipv6_connectivity -s

	### ipv4.

	touch "${nodes_db_ipv4_path}"

	if [ -s "${nodes_db_ipv4_path}" ]
	then

		### file is not empty, alter first line.

		sed -i "1s/.*/${public_ipv4}/" "${nodes_db_ipv4_path}"

	else

		### file is empty, write to it.

		echo "${public_ipv4}" > "${nodes_db_ipv4_path}"

	fi

	### ipv6

	touch "${nodes_db_ipv6_path}"

	if [ -s "${nodes_db_ipv6_path}" ]
	then

		### file is not empty, alter first line.

		if [ "${ipv6_available}" = "y" ]
		then
			sed -i "1s/.*/${this_ipv6}/" "${nodes_db_ipv6_path}"
		else
			sed -i "1s/.*/null/" "${nodes_db_ipv6_path}"
		fi

	else

		### file is empty, write to it.

		if [ "${ipv6_available}" = "y" ]
		then
			echo "${this_ipv6}" > "${nodes_db_ipv6_path}"
		else
			echo "null" > "${nodes_db_ipv6_path}"
		fi

	fi

	aenigma-create_nodes_db_numbered

}

aenigma-get_ejabberd_cluster_size() {

	ejabberd_cluster_size="$(wc -l < ${nodes_db_ipv4_path})"

}

aenigma-wipe_old_ufw_cluster_rules() {

	### ########################################################################

	### remove any and all ufw rules related to old cluster nodes.

	### ########################################################################

	### only do this check if ufw has actually been initialized.

	if systemctl is-active --quiet ufw
	then

		### get "ufw allow from x.x.x.x" rules previously set by us.

		ufw show added | grep "ufw allow from" | grep -oE '((1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])' > "${tmp_dir}/ufw_previous_cluster_rules" || true

		### check every item from previous command output to see if those ipv4s
		### are still part of the cluster and if not remove those rules.

		while read line
		do

			if [ ! "${line}" = "" ]
			then

				if ! grep -qr "${line}" "${conf_cluster_nodes_dir}"
				then

					ufw --force delete "$(ufw status | grep -E ' (ALLOW|LIMIT) ' | grep -n ${line} | cut -f1 -d":")"

				fi

			fi

		done < "${tmp_dir}/ufw_previous_cluster_rules"

		unset line

		rm "${tmp_dir}/ufw_previous_cluster_rules"

	fi

	############################################################################

}

aenigma-update_routing_info() {

	### wipe any old cluster-related ufw rule.

	aenigma-wipe_old_ufw_cluster_rules

	### set ufw rules on this node according to the updated nodes_db.

	array_iter=1
	IFS=$'\n'
	set -f
	for node_ipv4 in $(cat < "${nodes_db_ipv4_path}")
	do
		/usr/sbin/ufw insert "${array_iter}" allow from "${node_ipv4}"
		array_iter=$((array_iter + 1))
	done
	unset node_ipv4 array_iter
	IFS=$' \t\n'
	set +f
	echo

	### import nodes_db to /etc/hosts to provide ejabberd with a locally
	### resolved list of nodes to cluster with.

	if ! grep -q "### aenigma nodes list start" "/etc/hosts"
	then
		echo >> /etc/hosts
		echo "### aenigma nodes list start" >> "/etc/hosts"
	fi

	if ! grep -Pzoq '### aenigma nodes list start(?:.*\n)*?\K### aenigma nodes list end' "/etc/hosts"
	then
		echo "### aenigma nodes list end" >> "/etc/hosts"
	fi

	sed -i '/### aenigma nodes list start/,/### aenigma nodes list end/{//!d}' "/etc/hosts"
	sed -i "/### aenigma nodes list start/ r ${nodes_db_numbered_path}" "/etc/hosts"

}

aenigma-refresh_nodes_db_online() {

	### ########################################################################

	### this function lays out the directory structure for the execution of
	### get/test_nodes_online functions.
	### it will create a copy of nodes_db_ipv4 and nodes that appear to be
	### offline will be removed from it, for use by subsequent functions, until
	### the end of the clusterize script.

	### ########################################################################

	### touch temp dir.

	dna-touch_tmp_project_dir

	if [ -f "${nodes_db_online_path}" ]
	then
		rm "${nodes_db_online_path}"
		touch "${nodes_db_online_path}"
	fi

	cp "${nodes_db_ipv4_path}" "${nodes_db_online_path}"

}

aenigma-get_nodes_online() {

	### ########################################################################

	### generate a list of online nodes by opening SSH sessions to other nodes.
	### if any nodes are offline, remove them from nodes_db_online.
	### this is a routine function that gets called automatically before any
	### remote operation by the primary node, which simply temporarily excludes
	### offline nodes from those contacted by the primary node during the
	### installation, without in any way affecting nodes_db permanently,
	### and is mostly intented to be non-interactive, as opposed to
	### get_nodes_onlines.

	### ########################################################################

	aenigma-refresh_nodes_db_online

	array_iter=1
	IFS=$'\n'
	set -f

	for node_ipv4 in $(cat < "${nodes_db_ipv4_path}")
	do

		current_node_number="$(echo ${array_iter} | xargs printf %02d)"

		### ensure we don't try connecting to ourselves [therefore ignore the
		### first line of nodes_db_ipv4].

		if [ ! "${node_ipv4}" = "${public_ipv4}" ]
		then

			if ! ssh -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${node_ipv4}" exit
			then

				echo

				dna-echo -e "Existing node number | ${n-}${current_node_number}${r-} | is not responding at IPv4 address | ${n-}${node_ipv4}${r-} |."
				dna-echo -m "It is NOT reccommended to continue the installation as the node in question will not be informed of the new node we're setting up right now."
				dna-echo -m "However, you can continue running the installation now and re-run the installation script at at a later time."

				dna-ask_for_boolean -d "y" -q "Continue anyway?"

				if [ "${boolean_output}" = "y" ]
	            then

					dna-echo -n -m "Ok, continuing..."

					### remove node from nodes_db_online.

					sed -i "/^${node_ipv4}$/d" "${nodes_db_online_path}"

				else

					dna-echo -m "Ok, aborting..."
					dna-exit

				fi

			fi

		fi

		array_iter=$((array_iter + 1))

	done

	unset node_ipv4
	IFS=$' \t\n'
	set +f

}

aenigma-test_nodes_online() {

	### ########################################################################

	### check to see if any existing secondary nodes are offline, and, if so,
	### prompt to remove them from nodes_db_online.
	### this is meant as a specifically interactive and user-called function
	### to perform one-time cluster node removal operations, as opposed to
	### get_nodes_online.

	### ########################################################################

	### evaluate arguments.

	### ########################################################################

	while getopts ":as" arguments
	do
	    case $arguments in
			a)	test_nodes_online_automated="y"
			   ;;
	        s)	test_nodes_online_silent="y"
	            ;;
	        \?)	dna-echo -e "Invalid option | ${n-}-${OPTARG}${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
	            dna-exit
	            ;;
	    esac
	done
	OPTIND=1

	### ########################################################################

	if [ -z "${test_nodes_online_automated-}" ]
	then

		dna-echo -m "This function attempts to connect to every node in your cluster, and checks for consistency across all nodes' ejabberd's cluster synchronization state."
		dna-echo -m "If a node is found to be offline, you will have the choice to remove it from your primary node [this machine]'s database of nodes."
		dna-press_to_continue

	fi

	aenigma-refresh_nodes_db_online

	array_iter=1
	IFS=$'\n'
	set -f

	for node_ipv4 in $(cat < "${nodes_db_ipv4_path}")
	do

		current_node_number="$(echo ${array_iter} | xargs printf %02d)"

		### ensure we don't try connecting to ourselves [therefore ignore the
		### first line of nodes_db_ipv4].

		if [ ! "${node_ipv4}" = "${public_ipv4}" ]
		then

			if ssh -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${node_ipv4}" exit
			then

				if [ -z "${test_nodes_online_silent-}" ]
				then

					dna-echo -s "Node number: | ${n-}${current_node_number}${g-} | is responding at IPv4 address: | ${n-}${node_ipv4}${g-} |."

					aenigma-get_ejabberd_cluster_size

					if current_node_ejabberd_cluster_size="$(ssh -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${node_ipv4}" "ejabberdctl list_cluster | wc -l")" && [ "${current_node_ejabberd_cluster_size}" = "${ejabberd_cluster_size}" ]
					then

						dna-echo -s "ejabberd cluster status on node number: | ${n-}${current_node_number}${g-} | is nominal."

					else

						dna-echo -e "ejabberd cluster status on node number: | ${n-}${current_node_number}${g-} | is off-nominal."
						dna-echo -m "In order to fix this simply re-run the: | ${l-}clusterize${n-} | script once again after exiting this one."

					fi

				fi

			else

				echo

				dna-echo -e "Node number: | ${n-}${current_node_number}${r-} | is NOT responding at IPv4 address | ${n-}${node_ipv4}${r-} |."
				dna-echo -m "If the public IPv4 of such machine has changed, you will have to reinstall it again as a new node, after having removed it now."
				dna-echo -m "If you have decommissioned this server or intend to do so, you can remove this node from this primary node's configuration about the cluster."
				dna-echo -m "In either way, data on such secondary node will not be affected in any way, but this primary node won't consider it part of the cluster during future operations."
				dna-echo -m "If somehow such secondary node is still running, even though we cannot reach it right now, it will still actually take part in the ejabberd cluster."

	            dna-ask_for_boolean -d "n" -q "Would you like to remove this node from the ${project_name} cluster database now?"

	            if [ "${boolean_output}" = "y" ]
	            then

					dna-echo -m "Ok, removing node now..."

					/usr/sbin/ejabberdctl --no-timeout leave_cluster "ejabberd@ae${current_node_number}.${domain}" || true

					dna-echo -m "Now restarting ejabberd..."

					/usr/sbin/service ejabberd restart

					initial_last_node_number=$(wc -l < "${nodes_db_ipv4_path}" | xargs printf %02d)

					if [ "${current_node_number}" = "${initial_last_node_number}" ]
					then

						### remove node from nodes_db files.

						sed -i "${current_node_number}d" "${nodes_db_ipv4_path}"
						sed -i "${current_node_number}d" "${nodes_db_ipv6_path}"

						aenigma-create_nodes_db_numbered
						aenigma-push_routing_info

						dna-echo -s "Node deleted successfully."

						node_removed="y"

					else

						dna-echo -m "Performing this removal would leave an empty space for node number | ${o-}${current_node_number}${n-} |."
						dna-echo -m "To remove it from our cluster configuration it is required to reconfigure the last numbered node: [number | ${o-}${initial_last_node_number}${n-} |] to take the place of the newly removed node."
						dna-echo -m "If you do not want to proceed with this operation, the removal will be aborted."

						dna-ask_for_boolean -d "y" -q "Would you like to reconfigure node number | ${o-}${initial_last_node_number}${n-} | now?"

						if [ "${boolean_output}" = "y" ]
						then

							dna-echo -m "Ok, reconfiguring node now..."

							/usr/sbin/ejabberdctl --no-timeout leave_cluster "ejabberd@ae${initial_last_node_number}.${domain}" || true

							dna-echo -m "Now restarting ejabberd..."

							/usr/sbin/service ejabberd restart

							### get the last node in nodes_db_ipv4.

							target_node_ipv4=$(cat ${nodes_db_ipv4_path} | sed -n "${initial_last_node_number}"p)

							### copy its ipv4 to the line of the node to be
							### removed.

							sed -i "${current_node_number}s/.*/${target_node_ipv4}/" "${nodes_db_ipv4_path}"

							### delete what previously was the
							### reconfigured node's line.

							sed -i "${initial_last_node_number}d" "${nodes_db_ipv4_path}"

							### get the last node in nodes_db_ipv6.

							target_node_ipv6=$(cat ${nodes_db_ipv6_path} | sed -n "${initial_last_node_number}"p)

							### copy its IPv6 to the line of the node to be
							### removed.

							sed -i "${current_node_number}s/.*/${target_node_ipv6}/" "${nodes_db_ipv6_path}"

							### delete what previously was the
							### reconfigured node's line.

							sed -i "${initial_last_node_number}d" "${nodes_db_ipv6_path}"

							aenigma-create_nodes_db_numbered

							dna-echo -s "Node number: | ${n-}${initial_last_node_number}${g-} | successfully changed to number: | ${n-}${current_node_number}${g-} |."

							node_removed="y"
							launch_node_reinstallation="y"

						else

							dna-echo -m "Ok, aborting node removal."

						fi

					fi

					if [ "${launch_node_reinstallation-null}" = "y" ]
					then

						dna-echo -m "Having re-ordered what is now node number | ${o-}${current_node_number}${n-} | it's time to re-configure it and notify it of the change."
						dna-echo -m "Open a new shell to it [located at IP: | ${y-}${target_node_ipv4}${n-} |] and run the installer from there now. Then come back here."

						dna-press_to_continue

						dna-echo -n -m "Ok, now launching | ${c-}${project_name}-clusterize${n-} | script..."

						bash "${tools_dir}/${project_name}-clusterize"

					fi

				else

					dna-echo -m "Ok, leaving node registered in cluster database for now."
					dna-exit

				fi

			fi

		fi

		array_iter=$((array_iter + 1))

		if [ "${node_removed}" = "y" ]
		then

			dna-echo -e "Interrupting | ${n-}test_nodes_online${r-} | function execution."
			dna-echo -m "Having removed a node, if you want to keep testing online nodes, given the new node arrangement, simply re-run the script."

			dna-press_to_continue
			break

		fi

	done

	unset node_ipv4
	IFS=$' \t\n'
	set +f

	############################################################################

	dna-echo -m "Now pushing routing info..."

	aenigma-push_routing_info

	############################################################################

}

aenigma-check_ip_info_current() {

	if [ ! -f "${nodes_db_ipv4_path}" ] || [ ! -f "${nodes_db_ipv6_path}" ]
    then
		dna-echo -m "Not all nodes_db files found or none at all. Skipping IP change checks."
        return 0
    fi

	### Fetch IP addresses for this machine

	dna-get_public_ipv4

	dna-check_ipv6_connectivity

	### Check to see if this machine's IPs have changed since last install / clusterization

	this_registered_ipv4="$(cat ${nodes_db_ipv4_path} | head -1 | xargs)"

	this_registered_ipv6="$(cat ${nodes_db_ipv6_path} | head -1 | xargs)"

	if [ ! "${public_ipv4}" = "${this_registered_ipv4}" ]
	then
		dna-echo -e "The public IPv4 address for this machine appears to have changed since the last install or the last run of the clusterize script."
		dna-echo -m "It now appears to be: | ${y-}${public_ipv4}${n-} |."
		dna-echo -m "This node, however, appears to be registered with the following: | ${y-}${this_registered_ipv4}${n-} |."
		dna-echo -m "Please re-run the installation to update the records for this machine on the nodes database."
		dna-exit
	fi

	if [ ! "${this_ipv6}" = "${this_registered_ipv6}" ]
	then
		dna-echo -e "The public IPv6 address for this machine appears to have changed since the last install or the last run of the clusterize script."
		dna-echo -m "It now appears to be: | ${y-}${this_ipv6}${n-} |."
		dna-echo -m "This node, however, appears to be registered with the following: | ${y-}${this_registered_ipv6}${n-} |."
		dna-echo -m "Please re-run the installation to update the records for this machine on the nodes database."
		dna-exit
	fi

}

aenigma-push_routing_info() {

	############################################################################

	### Push nodes_db files to all secondary nodes for /etc/hosts and ufw rules configuration.
	### At the same time, check to see if any existing secondary nodes are offline, and, if so, remove them from nodes_db_online.

	aenigma-refresh_nodes_db_online

	array_iter=1
	IFS=$'\n'
	set -f

	for node_ipv4 in $(cat < "${nodes_db_ipv4_path}")							### Do NOT escape this variable
	do
		current_node_number="$(echo ${array_iter} | xargs printf %02d)"
		### Ensure we don't try connecting to ourselves [therefore ignore the first line of nodes_db_ipv4].
		if [ ! "${node_ipv4}" = "${public_ipv4}" ]
		then
			if ! /usr/bin/scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i /root/.ssh/aenigma_cluster -r "${conf_cluster_nodes_dir}" "root@${node_ipv4}:${conf_cluster_dir}"
			then
				echo
				dna-echo -e "Existing node number | ${n-}${current_node_number}${r-} | is not responding at IPv4 address | ${n-}${node_ipv4}${r-} |."
				dna-echo -m "It is NOT reccommended to continue the installation as the node in question will not be informed of the new node we're setting up right now."
				dna-echo -m "However, you can continue running the installation now and run the | ${o-}push_routing_info${n-} | script at at a later time by executing:"
				dna-echo -m " | ${y-}bash ${tools_dir}/${project_name}-push_routing_info${n-} | "
	            dna-ask_for_boolean -d "y" -q "Continue anyway?"
	            if [ "${boolean_output}" = "y" ]
	            then
					dna-echo -n -m "Ok, continuing..."
					### Remove node from nodes_db_online
					sed -i "/^${node_ipv4}$/d" "${nodes_db_online_path}"
				else
					dna-echo -m "Ok, aborting..."
					dna-exit
				fi
			fi
		fi
		array_iter=$((array_iter + 1))
	done
	unset node_ipv4
	IFS=$' \t\n'
	set +f

	############################################################################

	aenigma-exec() {

		### Define the commands that will be passed to the aenigma-exec script on the remote machine to be run, one per line between the quotes.

        bash openspace42/aenigma/tools/aenigma-exec -c "
aenigma-update_routing_info
"

	}

	############################################################################

	for online_node_ipv4 in $(cat < "${nodes_db_online_path}")
	do
		### Ensure we don't try connecting to ourselves [therefore ignore the first line of nodes_db_ipv4].
		if [ ! "${online_node_ipv4}" = "${public_ipv4}" ]
		then
			### SSH into the secondary node, declare the aenigma-exec function, and run it inside of the subshell.
	        ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${online_node_ipv4}" "$(declare -f aenigma-exec); export -f aenigma-exec; exec bash -li" <<END
set -eu
. /root/openspace42/aenigma/functions
synthia-source_dna
synthia-define_vars
dna-read_conf_settings
aenigma-exec
exit
END
		fi
		echo
	done
	unset online_node_ipv4

	############################################################################

}

aenigma-push_certs() {

	############################################################################

	### Push LetsEncrypt TLS certificate files to all secondary nodes.
	### At the same time, check to see if any existing secondary nodes are offline, and, if so, remove them from nodes_db_online.

	aenigma-refresh_nodes_db_online

	array_iter=1
	IFS=$'\n'
	set -f

	for node_ipv4 in $(cat < "${nodes_db_ipv4_path}")							### Do NOT escape this variable
	do
		current_node_number="$(echo ${array_iter} | xargs printf %02d)"
		### Ensure we don't try connecting to ourselves [therefore ignore the first line of nodes_db_ipv4].
		if [ ! "${node_ipv4}" = "${public_ipv4}" ]
		then
			if ! scp -P42022 -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i /root/.ssh/aenigma_cluster -r "${tls_project_dir}" "root@${node_ipv4}:${tls_dir}"
			then
				echo
				dna-echo -e "Node number | ${n-}${current_node_number}${r-} | is not responding at IPv4 address | ${n-}${node_ipv4}${r-} |."
				dna-echo -m "Ensure such node is actually online and when it is execute this script again by running:"
				dna-echo -m " | ${y-}${tools_dir}/${project_name}-create-push-certs${n-} | "
				if [ "${interactive}" = "y" ]
				then
		            dna-ask_for_boolean -d "y" -q "Continue anyway with all other nodes now?"
		            if [ "${boolean_output}" = "y" ]
		            then
						dna-echo -n -m "Ok, continuing..."
						### Remove node from nodes_db_online
						sed -i "/^${node_ipv4}$/d" "${nodes_db_online_path}"
					else
						dna-echo -m "Ok, aborting..."
						dna-exit
					fi
				else
					dna-echo -e "Skipping node | ${n-}${current_node_number}${r-} | and continuing with all others now."
					### Send an alert email.
			        dna-send_mail -s "aenigma: node: | ${current_node_number} | offline" \
			        -b "Automated post-renewal LetsEncrypt TLS certificate files push to node: | ${current_node_number} | has failed, as it appears to be offline. Look into the situation and when the node is back online simply log into your primary node [the machine sending this very notification] and execute: | ${tools_dir}/${project_name}-create-push-certs |." \
			        -r "${admin_mail}"
				fi
			fi
		fi
		array_iter=$((array_iter + 1))
	done
	unset node_ipv4
	IFS=$' \t\n'
	set +f

	############################################################################

	aenigma-exec() {

		### Define the commands that will be passed to the aenigma-exec script on the remote machine to be run, one per line between the quotes.

        bash openspace42/aenigma/tools/aenigma-exec -c "
set +e
chown -R "ejabberd:ejabberd" "${tls_project_dir}"
set -e
chmod -R 700 "${tls_project_dir}"
aenigma-restart_ejabberd_if_running
"

	}

	############################################################################

	for online_node_ipv4 in $(cat < "${nodes_db_online_path}")
	do
		### Ensure we don't try connecting to ourselves [therefore ignore the first line of nodes_db_ipv4].
		if [ ! "${online_node_ipv4}" = "${public_ipv4}" ]
		then
			### SSH into the secondary node, declare the aenigma-exec function, and run it inside of the subshell.
	        ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${online_node_ipv4}" "$(declare -f aenigma-exec); export -f aenigma-exec; exec bash -li" <<END
set -eu
. /root/openspace42/aenigma/functions
synthia-source_dna
synthia-define_vars
dna-read_conf_settings
aenigma-exec
exit
END
		fi
		echo
	done
	unset online_node_ipv4

	############################################################################

}

aenigma-create_push_certs() {

	### evaluate arguments.

	### ########################################################################

	while getopts ":e" arguments
	do
		case "${arguments}" in
			e)	skip_ejabberd_stop_start_operations="y"
				;;
			\?)	dna-echo -e "Invalid option | ${n-}-$OPTARG${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
				dna-exit
				;;
		esac
	done
	OPTIND=1

	### ########################################################################

	### ensure that this is the primary node in the event that cluster
	### is enabled.

	############################################################################

	if [ "${cluster_mode-null}" = "secondary" ]
	then
		dna-echo -e "This script must only be run on the primary node of your ${project_name} instance."
		dna-exit
	fi

	############################################################################

	if [ ! -f "${nodes_db_ipv4_path}" ]
	then
		dna-echo -e " | ${n-}nodes_db${r-} | file not found."
		dna-echo -m "Either the initial installation was never completed or the file was somehow removed."
		dna-echo -m "Re-run the installation to fix this."
		dna-exit
	fi

	############################################################################
	############################################################################

	### ensure public IPs are still the same
	### as when the installer ran the last time.

	############################################################################

	aenigma-check_ip_info_current

	############################################################################
	############################################################################

	### ensure $tls_project_dir exists.

	############################################################################

	mkdir -p "${tls_project_dir}"

	############################################################################
	############################################################################

	### provision the LetsEncrypt wildcard TLS certificate for $domain
	### and each $additional_domains item.

	############################################################################

	dna-echo -m "It's time to set up the LetsEncrypt wildcard TLS [SSL] certificate that will be valid for | *.${domain} | and for any additional domains you specified."

	for this_domain in $(cat < "${domains_db_path}")
	do

	    dna-provision_le_wildcard_cert -d "${this_domain}"

	done

	############################################################################
	############################################################################

	### create all-in-one TLS certs for ejabberd.

	############################################################################

	dna-echo_operation -h "creating all-in-one TLS file from LetsEncrypt wildcard cert for ejabberd"

	for le_cert_domain_full in "$le_live_dir"/*
	do

	    le_cert_domain="$(echo ${le_cert_domain_full} | sed 's:.*/::')"

	    if [[ "${domain}" == "${le_cert_domain}" ]] || [[ "${additional_domains}" == *"${le_cert_domain}"* ]]
	    then

	        ####################################################################

	        dna-echo -m "Now creating all-in-one TLS certificate file for ${le_cert_domain}..."

	        touch "${tls_project_dir}/${le_cert_domain}.pem"

	        cat "${le_live_dir}/${le_cert_domain}/privkey.pem" > "${tls_project_dir}/${le_cert_domain}.pem"
	        cat "${le_live_dir}/${le_cert_domain}/fullchain.pem" >> "${tls_project_dir}/${le_cert_domain}.pem"

	        dna-echo -s "Finished creating all-in-one TLS certificate file for ${le_cert_domain}."

	        ####################################################################

	        dna-echo -m "Now copying individual fullchain and privkey files for ${le_cert_domain}..."

	        mkdir -p "${tls_project_dir}/${le_cert_domain}.d/"

	        cp "${le_live_dir}/${le_cert_domain}/privkey.pem" "${tls_project_dir}/${le_cert_domain}.d/"
	        cp "${le_live_dir}/${le_cert_domain}/fullchain.pem" "${tls_project_dir}/${le_cert_domain}.d/"

	        dna-echo -s "Finished copying individual fullchain and privkey files for ${le_cert_domain}."

	        ####################################################################
			####################################################################

	    fi

	done

	############################################################################
	############################################################################

	### set permissions.

	############################################################################

	chown -R "ejabberd:ejabberd" "${tls_project_dir}" 2> /dev/null || true
	chmod -R 700 "${tls_project_dir}"

	dna-echo_operation -t

	############################################################################
	############################################################################

	### Verify output

	############################################################################

	if [ -f "${tls_project_dir}/${domain}.pem" ]
	then
	    dna-echo -s "TLS certificate for ${domain} found in | ${n-}${tls_project_dir}/${domain}.pem${g-} |."
	    sleep 1
	else
	    dna-echo -e "Something went wrong provisioning the wildcard TLS certificate for ${domain}."
	    dna-exit
	fi

	################################################################################
	############################################################################

	if [ ! "${psql_migration_required-null}" = "true" ] && [ -z "${skip_ejabberd_stop_start_operations-}" ]
	then

	    aenigma-restart_ejabberd_if_running

	fi

	############################################################################
	############################################################################

	### push certs to other nodes.

	############################################################################

	aenigma-push_certs

	############################################################################
	############################################################################

	unset skip_ejabberd_stop_start_operations

	############################################################################
	############################################################################

}

aenigma-perform_dns_checks() {

	### Perform DNS checks. They must only be run by the primary node.

	############################################################################

	dna-echo -m "Performing DNS checks for the following domains:"
	dna-echo -m "${y-}$(cat ${domains_db_path})"

	if [ "${cluster_enabled}" = "cluster" ]
	then

		dna-echo -m "Performing DNS checks for the following nodes:"
		dna-echo -m "${y-}$(cat ${nodes_db_numbered_path})"

	fi

	############################################################################

	for this_domain in $(cat < "${domains_db_path}")
	do

		### Perform checks in both IPv4 and IPv6. If this node [or a node in case of a cluster] doesn't have IPv6 connectivity its IPv6 check will be ignored locally.

		declare -a ip_version_array=(
			"4"
			"6"
		)

		aenigma-calculate_ip_version_vars() {

			if [ "${ip_version}" = "4" ]
			then
				nodes_db_path="${nodes_db_ipv4_path}"
				address_type="a"
			elif [ "${ip_version}" = "6" ]
			then
				nodes_db_path="${nodes_db_ipv6_path}"
				address_type="aaaa"
			fi

		}

		########################################################################

		### Check hostnames.

		for ip_version in "${ip_version_array[@]}"
		do

			aenigma-calculate_ip_version_vars

			array_iter=1
			IFS=$'\n'
			set -f

			for node_ip in $(cat < "${nodes_db_path}")
			do

				current_node_number="$(echo ${array_iter} | xargs printf %02d)"
				dna-check_dns_record -n "ae${current_node_number}.${this_domain}" -t "${address_type}" -r "${node_ip}"
				array_iter=$((array_iter + 1))

			done

			unset array_iter
			IFS=$' \t\n'
			set +f

		done

		########################################################################

		### Now check that "ae.$this_domain" points to all nodes in a round-robin fashion.

		for ip_version in "${ip_version_array[@]}"
		do

			aenigma-calculate_ip_version_vars

			dna-check_dns_record -n "ae.${this_domain}" -t "${address_type}" -r "${nodes_db_path}"

		done

		########################################################################

		### Check XMPP services records

		### First check services that need to point to each node respectively

		for ip_version in "${ip_version_array[@]}"
		do

			aenigma-calculate_ip_version_vars

			array_iter=1
			IFS=$'\n'
			set -f

			for node_ip in $(cat < "${nodes_db_path}")
			do

				current_node_number="$(echo ${array_iter} | xargs printf %02d)"

				declare -a xmpp_service_array=(
					"xh"
					"xm"
					"xp"
					"xr"
					"xv"
				)

				for xmpp_service in "${xmpp_service_array[@]}"
				do

					dna-check_dns_record -n "${xmpp_service}${current_node_number}.${this_domain}" -t "${address_type}" -r "${node_ip}"

				done

				array_iter=$((array_iter + 1))

			done

			unset array_iter
			IFS=$' \t\n'
			set +f

		done

		### Then check "xc.$this_domain" as it should not be a set of numbered records [xc01, xc02...] but instead be only one record pointing to all nodes in a round-robin fashion

		for ip_version in "${ip_version_array[@]}"
		do

			aenigma-calculate_ip_version_vars

			dna-check_dns_record -n "xc.${this_domain}" -t "${address_type}" -r "${nodes_db_path}"

		done

		########################################################################

		### Check SRV records

		#### SRV client records

		dna-check_dns_record -n "_xmpps-client._tcp.${this_domain}" -t "srv" -r "5 1 5223 ae.${this_domain}.,15 1 443 ae.${this_domain}."
		dna-check_dns_record -n "_xmpp-client._tcp.${this_domain}" -t "srv" -r "10 1 5222 ae.${this_domain}.,20 1 443 ae.${this_domain}."

		#### SRV server records

		declare -a srv_record_array=(
			"_jabber._tcp"
			"_xmpp-server._tcp"
		)

		for srv_record in "${srv_record_array[@]}"
		do

			dna-check_dns_record -n "${srv_record}.${this_domain}" -t "srv" -r "0 0 5269 ae.${this_domain}."

		done

		########################################################################

		unset dns_checks_outcome

		########################################################################

	done

}

################################################################################
################################################################################

### Postgresql

aenigma-install_postgresql_dependencies() {

	dna-install_dependencies -p "python python-pip python-psycopg2 libpq-dev"
	pip install --upgrade setuptools
	pip install python-etcd
	pip uninstall --yes requests || true
	pip install requests
	pip install docopt
	echo

}

aenigma-wipe_etcd() {

	### Stop etcd if it's already installed

	if dpkg -s etcd &> /dev/null
	then
		service etcd stop || true
	fi

	### Purge etcd

	apt-get -y purge etcd

	### Delete any previous etcd data in case it gets left over by apt

	if [ -d "/var/lib/etcd/default/" ]
	then
		rm -r "/var/lib/etcd/default/"
	fi

}

aenigma-install_etcd() {

	############################################################################

	### Install etcd

	### !!! This function is only intended to be ran by the primary node !!! ###

	############################################################################
	############################################################################

	### Install dependencies required for postgresql cluster

	aenigma-install_postgresql_dependencies

	### Apply permissions

	aenigma-set_permissions

	############################################################################
	############################################################################

	### Delete all etcd data to start fresh

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		########################################################################

		### This only applies to primary nodes, so that an initial wipe will
		### occur only once at the same time during primary node installation.
		### Secondary nodes will skip this when running on their own as the
		### primary node will already have remotely performed this onto them as
		### well.

		aenigma-wipe_etcd

		### Delete any previous etcd data on secondary nodes

		########################################################################

		aenigma-exec() {

			### Define the commands that will be passed to the aenigma-exec script on the remote machine to be run, one per line between the quotes.

	        bash openspace42/aenigma/tools/aenigma-exec -c "
aenigma-wipe_etcd
"

		}

		########################################################################

		for online_node_ipv4 in $(cat < "${nodes_db_online_path}")
		do
			### Ensure we don't try connecting to ourselves [therefore ignore the first line of nodes_db_ipv4].
			if [ ! "${online_node_ipv4}" = "${public_ipv4}" ]
			then
				### SSH into the secondary node, declare the aenigma-exec function, and run it inside of the subshell.
		        ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${online_node_ipv4}" "$(declare -f aenigma-exec); export -f aenigma-exec; exec bash -li" <<END
set -eu
. /root/openspace42/aenigma/functions
synthia-source_dna
synthia-define_vars
dna-read_conf_settings
aenigma-exec
exit
END
			fi
			echo
		done
		unset online_node_ipv4

		########################################################################

	else

		### This is a secondary node, wipe etcd locally before localizing it
		### and starting it back up.

		aenigma-wipe_etcd

	fi

	############################################################################
	############################################################################

	### Perform the installation

	############################################################################

	### Inhibit etcd from starting after install

	echo exit 101 > "/usr/sbin/policy-rc.d"
	chmod +x "/usr/sbin/policy-rc.d"

	### Proceed

	dna-install_etcd

	### Remove autostart inhibition

	rm "/usr/sbin/policy-rc.d"

	### Add etcd user to aenigma group for correct tls_project_dir permissions

	usermod -aG aenigma etcd

	############################################################################
	############################################################################

	### Set config file

	cat <<EOF > "/etc/default/etcd"
ETCD_HEARTBEAT_INTERVAL="100"
ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="https://public_ipv4-var:2380"
ETCD_LISTEN_CLIENT_URLS="http://localhost:2379,http://public_ipv4-var:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://public_ipv4-var:2380"
ETCD_INITIAL_CLUSTER="hostname-var=https://hostname-var:2380,"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="aenigma"
ETCD_ADVERTISE_CLIENT_URLS="http://public_ipv4-var:2379"
ETCD_PEER_CERT_FILE="/etc/ssl/aenigma/domain-var.d/fullchain.pem"
ETCD_PEER_KEY_FILE="/etc/ssl/aenigma/domain-var.d/privkey.pem"
#ETCD_CERT_FILE="/etc/ssl/aenigma/domain-var.d/fullchain.pem"
#ETCD_KEY_FILE="/etc/ssl/aenigma/domain-var.d/privkey.pem"
EOF

	### Add cluster nodes to config file

	aenigma-get_ejabberd_cluster_size

	if [ ! "${ejabberd_cluster_size}" = "1" ]
	then

		for node in $(seq 1 $ejabberd_cluster_size)
		do

			node_number_padded="$(echo ${node} | xargs printf %02d)"

			if [ ! "${node_number_padded}" = "${node_number}" ]
			then

				sed -i "6s/.$/ae${node_number_padded}.${domain}=https:\/\/ae${node_number_padded}.${domain}:2380,&/" "/etc/default/etcd"

			fi

		done

	fi

	### Replace vars in config file

	dna-get_hostname

	sed -i "s|hostname-var|${hostname}|g" "/etc/default/etcd"
	sed -i "s|domain-var|${domain}|g" "/etc/default/etcd"
	sed -i "s|public_ipv4-var|${public_ipv4}|g" "/etc/default/etcd"

	dna-echo_operation -h "starting etcd on node: | ${l-}${hostname}${n-} |"

	update-rc.d etcd enable

	systemctl restart etcd &

	sleep 3

	dna-wait -s "8" -d "Now allowing etcd to start up correctly..."

	dna-echo_operation -t

	############################################################################
	############################################################################

}

aenigma-install_etcd_cluster() {

	############################################################################

	### Install etcd all at once on all nodes

	############################################################################

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		########################################################################

		### Ensure routing info is up-to-date on all nodes or else etcd will
		### fail.

		aenigma-push_routing_info

		### Ensure no old nodes can connect to any current nodes or else their
		### etcd might connect to the latter and make etcd fail

		aenigma-wipe_old_ufw_cluster_rules

		########################################################################

		### Stop ejabberd and patroni cluster-wide

		aenigma-stop_ejabberd_patroni_clusterwide

		########################################################################

		### Install etcd locally while performing a cluster-wide etcd wipe

		aenigma-install_etcd

		########################################################################

		### At this point etcd will be installed locally on this primary node
		### and we can proceed to install it on any secondary nodes.

		########################################################################

		aenigma-get_nodes_online

		aenigma-exec() {

			### Define the commands that will be passed to the aenigma-exec script on the remote machine to be run, one per line between the quotes.

			bash openspace42/aenigma/tools/aenigma-exec -c "
aenigma-install_etcd
"

		}

		########################################################################

		for online_node_ipv4 in $(cat < "${nodes_db_online_path}")
		do
			### Ensure we don't try connecting to ourselves [therefore ignore the first line of nodes_db_ipv4].
			if [ ! "${online_node_ipv4}" = "${public_ipv4}" ]
			then
				### SSH into the secondary node, declare the aenigma-exec function, and run it inside of the subshell.
				ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${online_node_ipv4}" "$(declare -f aenigma-exec); export -f aenigma-exec; exec bash -li" <<END
set -eu
. /root/openspace42/aenigma/functions
synthia-source_dna
synthia-define_vars
dna-read_conf_settings
aenigma-exec
exit
END
			fi
			echo
		done
		unset online_node_ipv4

		########################################################################

	fi

}

aenigma-install_etcd_cluster_conditionally() {

	aenigma-get_ejabberd_cluster_size

	if dpkg -s etcd &> /dev/null
	then

		if etcdctl cluster-health | grep -q "cluster is healthy" && ! etcdctl cluster-health | grep -q "failed"
		then

			etcd_cluster_size="$(etcdctl member list | wc -l)"

			if [ "${etcd_cluster_size}" = "${ejabberd_cluster_size}" ]
			then

				dna-echo -s "etcd appears to be already installed and cluster appears to be healty."
				dna-echo -m "Skipping re-install..."
				return 0

			fi

		fi

	fi

	dna-echo -m "etcd doesn't appear to be already installed or cluster isn't fully up and running."
	dna-echo -m "installing etcd now..."

	aenigma-install_etcd_cluster

}

aenigma-apply_and_restart_patroni_haproxy_ejabberd() {

	############################################################################

	### Run this function after re-installing or clusterizing
	### to apply new configs updated with new nodes to patroni and haproxy
	### and restart these two services plus ejabberd

	############################################################################

	while getopts ":e:r" arguments
	do
        case $arguments in
            e)	excluded_node="${OPTARG}"
	            ;;
			r)	remote_only="yes"
	            ;;
            \?)	dna-echo -e "Invalid option | ${n-}-${OPTARG}${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
	            dna-exit
	            ;;
        esac
    done
    OPTIND=1

	############################################################################
	############################################################################

	### Localhost

	############################################################################

	if [ ! "${remote_only-null}" = "yes" ]
	then

		### Update patroni config now that we have a new node

		aenigma-apply_patroni_config

		aenigma-start_patroni

		### Update haproxy config now that we have a new node

		aenigma-apply_haproxy_config

		### Restart ejabberd

		aenigma-start_ejabberd

	fi

	############################################################################
	############################################################################

	### Secondary nodes

	############################################################################

	aenigma-get_nodes_online

	aenigma-exec() {

		### Define the commands that will be passed to the aenigma-exec script on the remote machine to be run, one per line between the quotes.

		bash openspace42/aenigma/tools/aenigma-exec -c "
aenigma-apply_patroni_config
aenigma-start_patroni
aenigma-apply_haproxy_config
aenigma-start_ejabberd
"

	}

	############################################################################

	for online_node_ipv4 in $(cat < "${nodes_db_online_path}")
	do

		### Ensure we don't try connecting to ourselves [therefore ignore
		### the first line of nodes_db_ipv4] or to the excluded node.

		if [ ! "${online_node_ipv4}" = "${public_ipv4}" ] && [ ! "${online_node_ipv4}" = "${excluded_node-}" ]
		then

			### SSH into the secondary node, declare the aenigma-exec function,
			### and run it inside of the subshell.

			ssh -tt -p42022 -o "BatchMode=yes" -o "ConnectTimeout=8" -o "LogLevel=ERROR" -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" -i "/root/.ssh/aenigma_cluster" "root@${online_node_ipv4}" "$(declare -f aenigma-exec); export -f aenigma-exec; exec bash -li" <<END
set -eu
. /root/openspace42/aenigma/functions
synthia-source_dna
synthia-define_vars
dna-read_conf_settings
aenigma-exec
exit
END

		fi

		echo

	done

	unset online_node_ipv4

	############################################################################
	############################################################################

	unset excluded_node remote_only

	############################################################################
	############################################################################

}

aenigma-migrate-mnesia-postgresql() {

	### !!! Only run this after install_patroni_postgresql and on the primary node !!! ####

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		### Insert the new_sql_schema directive in a random point of the config without enabling SQL yet

		dna-echo_operation -h "applying new_sql_schema directive"

		sed -i "s|## fqdn: \"server3.example.com\"|new_sql_schema: true|g" "/opt/ejabberd/conf/ejabberd.yml"

		service ejabberd restart

		dna-echo_operation -t

		### Alert about migration

		dna-echo -m "We're now migrating this ejabberd instance from using the mnesia DB to PostgreSQL."
		dna-echo -m "We will now proceed with exporting and re-importing all ejabberd data on this instance."

		### Close firewall ports to effectively terminate the service for users and avoid collecting data that would not be migrated if generated after the export

		ufw delete allow 5222
		ufw delete allow 5223
		ufw delete allow 5444
		echo

		### Export data for every ejabberd domain

		dna-echo_operation -h "exporting data from the ejabberd mnesia database"

		ejabberdctl export2sql "${domain}" "aenigma-ejabberd-export-${domain}.sql"
		mv "/var/lib/ejabberd/aenigma-ejabberd-export-${domain}.sql" "${tmp_dir}/aenigma-ejabberd-export-${domain}.sql"
		chmod 777 "${tmp_dir}/aenigma-ejabberd-export-${domain}.sql"

		additional_domains_array=(${additional_domains})
		for additional_domain in "${additional_domains_array[@]}"
		do
			ejabberdctl export2sql "${additional_domain}" "aenigma-ejabberd-export-${additional_domain}.sql"
			mv "/var/lib/ejabberd/aenigma-ejabberd-export-${additional_domain}.sql" "${tmp_dir}/aenigma-ejabberd-export-${additional_domain}.sql"
			chmod 777 "${tmp_dir}/aenigma-ejabberd-export-${additional_domain}.sql"
		done

		dna-echo_operation -t

		### Import to psql

		dna-echo_operation -h "importing data into PSQL"

		for export in ${tmp_dir}/aenigma-ejabberd-export*.sql
		do
			PGPASSWORD="${patroni_ejabberd_password}" psql -h "${hostname}" -p "5432" -U ejabberd ejabberd < "${export}"
			echo
		done

		dna-echo_operation -t

		psql_migration_status="complete"

	fi

}

aenigma-get_postgresql_bin_dir_path() {

	postgresql_bin_dir_path="$(find /usr/lib/postgresql/ -name bin)"

}

aenigma-apply_patroni_config() {

	############################################################################

	### Install and localize patroni config

	############################################################################

	dna-get_hostname

	if dpkg -s python-pip &> /dev/null && pip list 2> /dev/null | grep -q patroni
	then

		########################################################################

		dna-echo_operation -h "applying patroni config on node: | ${l-}${hostname}${n-} |"

		########################################################################

		### Install patroni config

		cat <<EOF > "/etc/patroni.yml"
scope: aenigma
name: postgresql-node_number-var

restapi:
  listen: public_ipv4-var:8008
  connect_address: public_ipv4-var:8008
  certfile: /etc/ssl/aenigma/domain-var.d/fullchain.pem
  keyfile: /etc/ssl/aenigma/domain-var.d/privkey.pem
#  authentication:
#    username: username
#    password: password

ctl:
  insecure: false # Allow connections to SSL sites without certs
  certfile: /etc/ssl/aenigma/domain-var.d/fullchain.pem
  keyfile: /etc/ssl/aenigma/domain-var.d/privkey.pem

etcd:
  host: public_ipv4-var:2379

bootstrap:
  dcs:
    ttl: 30
    loop_wait: 10
    retry_timeout: 10
    maximum_lag_on_failover: 1048576
    postgresql:
      use_pg_rewind: true
      use_slots: true
      parameters:
        wal_level: replica
        hot_standby: "on"
        wal_keep_segments: 8
        max_wal_senders: 5
        max_replication_slots: 5
        checkpoint_timeout: 30

  initdb:
  - encoding: UTF8
  - locale: en_US.utf8
#  - data-checksums

  pg_hba:
  - host all postgres all md5
  - host replication repl all md5
  - host replication replicator public_ipv4-var/32 md5
  - host all all 0.0.0.0/0 md5

  users:
    postgres:
      password: patroni_postgres_password-var
      options:
        - createrole
        - createdb
    repl:
      password: patroni_repl_password-var
      options:
        - replication

postgresql:
  listen: public_ipv4-var:5432
  connect_address: public_ipv4-var:5432
  data_dir: /data/postgresql
  config_dir: /data/postgresql
  bin_dir: postgresql_bin_dir_path-var
  authentication:
    replication:
      username: repl
      password: patroni_repl_password-var
    superuser:
      username: postgres
      password: patroni_postgres_password-var
  parameters:
    unix_socket_directories: '/tmp'
EOF

		############################################################################

		### Add other cluster nodes to the patroni config

		for node_ipv4 in $(cat < "${nodes_db_ipv4_path}")
		do
			### Ensure we don't add ourselves [therefore ignore the first line of nodes_db_ipv4].
			if [ ! "${node_ipv4}" = "${public_ipv4}" ]
			then
				sed -i "/replicator public_ipv4-var/a \  - host replication replicator ${node_ipv4}\/0 md5" "/etc/patroni.yml"
			fi
		done
		unset node_ipv4

		############################################################################

		### Localize vars in patroni config

		sed -i "s|public_ipv4-var|${public_ipv4}|g" "/etc/patroni.yml"
		sed -i "s|node_number-var|${node_number}|g" "/etc/patroni.yml"
		sed -i "s|domain-var|${domain}|g" "/etc/patroni.yml"

		aenigma-get_postgresql_bin_dir_path

		sed -i "s|postgresql_bin_dir_path-var|${postgresql_bin_dir_path}|g" "/etc/patroni.yml"

		############################################################################

		### Apply passwords to the postgresql/patroni config

		sed -i "s|patroni_postgres_password-var|${patroni_postgres_password}|g" "/etc/patroni.yml"
		sed -i "s|patroni_repl_password-var|${patroni_repl_password}|g" "/etc/patroni.yml"

		############################################################################

		dna-echo_operation -t

		########################################################################

	else

		########################################################################

		dna-echo -m "patroni not [fully] installed on node: | ${l-}${hostname}${n-} |"
		dna-echo -m "skipping config apply for now."

		########################################################################

	fi

	############################################################################
	############################################################################

}

aenigma-install_postgresql_patroni() {

	############################################################################

	### Install postgresql and patroni

	############################################################################

	### Install dependencies required for postgresql cluster

	aenigma-install_postgresql_dependencies

	### Apply permissions

	aenigma-set_permissions

	############################################################################

	### Get hostname

	dna-get_hostname

	### Get public IPv4

	dna-get_public_ipv4

	############################################################################

	### Install postgresql

	dna-echo_operation -h "installing PostgreSQL"

	dna-install_postgresql

	### Install erlang-p1-pgsql

	dna-get_distro_name

	if [ "${distro_name}" = "debian" ]
	then
		dna-enable_debian_testing_repo
	fi

	dna-install_dependencies -p "erlang-p1-pgsql"

	### Link psql bins

	if [ -d "/usr/lib/postgresql/" ]
	then

		aenigma-get_postgresql_bin_dir_path

		ln -s ${postgresql_bin_dir_path}/* "/usr/sbin/" || true
		echo

	fi

	### Add postgres user to aenigma group for correct tls_project_dir permissions

	usermod -aG aenigma postgres

	### Finish

	dna-echo_operation -t

	############################################################################

	### Stop and disable psql as patroni will now be managing DBs

	if systemctl is-active --quiet postgresql
	then
		service postgresql stop
	fi

	if [ -f "/etc/init.d/postgresql" ]
	then
		systemctl disable postgresql
	fi

	############################################################################

	### Install patroni

	pip install patroni
	echo

	############################################################################

	### Generate random passwords for the postgresql/patroni config if this is a primary node

	############################################################################

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		### Generate a random password for the postgres psql user

		if [ -z "${patroni_postgres_password-}" ]
		then
			dna-echo -m "Generating password for the postgresql/patroni postgres user"
			patroni_postgres_password="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 24 | head -n 1)"
			dna-set_option -o "patroni_postgres_password" -p "postgresql" -s "${patroni_postgres_password}"
		fi

		### Generate a random password for the repl psql user

		if [ -z "${patroni_repl_password-}" ]
		then
			dna-echo -m "Generating password for the postgresql/patroni repl user"
			patroni_repl_password="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 24 | head -n 1)"
			dna-set_option -o "patroni_repl_password" -p "postgresql" -s "${patroni_repl_password}"
		fi

	fi

	############################################################################

	### Apply patroni config

	aenigma-apply_patroni_config

	############################################################################

	aenigma-set_permissions

	############################################################################

	### Create patroni systemd service

	cat <<EOF > "/etc/systemd/system/patroni.service"
[Unit]
Description=Runners to orchestrate a high-availability PostgreSQL
After=syslog.target network.target

[Service]
Type=simple

User=postgres
Group=postgres

ExecStart=/usr/local/bin/patroni /etc/patroni.yml

KillMode=process

TimeoutSec=30

Restart=no

[Install]
WantedBy=multi-user.target
EOF

	systemctl daemon-reload

	systemctl enable patroni

	systemctl start patroni

	echo

	############################################################################

	dna-echo_operation -h "starting patroni on node: | ${l-}${hostname}${n-} |"

	dna-wait -s "11" -d "Now allowing patroni to start up correctly..."

	dna-echo_operation -t

	############################################################################

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		########################################################################

		### Create the ejabberd role and database in postgresql/patroni

		########################################################################

		### Generate a random password for the ejabberd psql user

		if [ -z "${patroni_ejabberd_password-}" ]
		then
			dna-echo -m "Generating password for the ejabberd postgresql/patroni user"
			patroni_ejabberd_password="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 24 | head -n 1)"
			dna-set_option -o "patroni_ejabberd_password" -p "postgresql" -s "${patroni_ejabberd_password}"
		fi

		### Create ejabberd postgresql/patroni user

		if ! PGPASSWORD="${patroni_postgres_password}" psql -h "${hostname}" -p "5432" -U "postgres" -c "\du" 2> /dev/null | grep -q ejabberd
		then
			dna-echo -m "Creating the ejabberd postgresql/patroni user"
			PGPASSWORD="${patroni_postgres_password}" psql -h "${hostname}" -p "5432" -U "postgres" -c "create role ejabberd"
			echo
			patroni_ejabberd_role_previous_state="nx"
		fi

		### Assign the generated password to the user if we've just created it

		if [ "${patroni_ejabberd_role_previous_state-null}" = "nx" ]
		then
			dna-echo -m "Assigning password to the ejabberd postgresql/patroni user"
			PGPASSWORD="${patroni_postgres_password}" psql -h "${hostname}" -p "5432" -U "postgres" -c "alter user ejabberd with encrypted password '$patroni_ejabberd_password'"
			echo
		fi

		### Allow the ejabberd postgresql/patroni user to log in with password if we've just created it

		if [ "${patroni_ejabberd_role_previous_state-null}" = "nx" ]
		then
			dna-echo -m "Assigning password to the ejabberd postgresql/patroni user"
			PGPASSWORD="${patroni_postgres_password}" psql -h "${hostname}" -p "5432" -U "postgres" -c "alter role ejabberd with login"
			echo
		fi

		### Create the ejabberd postgresql/patroni database

		if ! PGPASSWORD="${patroni_postgres_password}" psql -h "${hostname}" -p "5432" -U "postgres" -c "\l" 2> /dev/null | grep -q ejabberd
		then
			dna-echo -m "Creating PSQL database"
			PGPASSWORD="${patroni_postgres_password}" psql -h "${hostname}" -p "5432" -U "postgres" -c "create database ejabberd"
			patroni_ejabberd_db_previous_state="nx"
		fi

		### Make the ejabberd postgresql/patroni user the owner of its database

		if [ "${patroni_ejabberd_role_previous_state-null}" = "nx" ]
		then
			dna-echo -m "Assigning password to the ejabberd postgresql/patroni user"
			PGPASSWORD="${patroni_postgres_password}" psql -h "${hostname}" -p "5432" -U "postgres" -c "alter database ejabberd owner to ejabberd;"
			echo
		fi

		########################################################################

		### Import ejabberd psql schema

		########################################################################

		### Import ejabberd psql schema if the db was just created

		if [ "${patroni_ejabberd_db_previous_state-null}" = "nx" ]
		then

			### Download ejabberd psql schema

			curl "https://raw.githubusercontent.com/processone/ejabberd/master/sql/pg.new.sql" > "${tmp_project_dir}/pg.new.sql"
			chmod 777 "${tmp_project_dir}/pg.new.sql"
			echo

			### Import into postgresql

			dna-echo -m "Importing PSQL default schema"
			PGPASSWORD="${patroni_ejabberd_password}" psql -h "${hostname}" -p "5432" -U ejabberd ejabberd < "/tmp/aenigma/pg.new.sql"
			echo

		fi

		########################################################################

		### Migrate mnesia to psql if necessary

		########################################################################

		if [ "${psql_migration_required-null}" = "true" ]
		then

			### Start ejabberd

			aenigma-start_ejabberd

			### This is an old instance, start the migration

			aenigma-migrate-mnesia-postgresql

		fi

		########################################################################

	fi

}

aenigma-apply_haproxy_config() {

	############################################################################

	### Apply haproxy config

	############################################################################

	cat <<EOF > "/etc/haproxy/haproxy.cfg"
global
    maxconn 100

defaults
    log global
    mode tcp
    retries 2
    timeout client 30m
    timeout connect 4s
    timeout server 30m
    timeout check 5s

listen stats
    mode http
    bind *:7000
    stats enable
    stats uri /

listen postgres
    bind *:5000
    option httpchk
    http-check expect status 200
    default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
    server postgresql_public_ipv4-var_5432 public_ipv4-var:5432 maxconn 100 check check-ssl verify none port 8008
EOF

	for node_ipv4 in $(cat < "${nodes_db_ipv4_path}")
	do
		### Ensure we don't add ourselves [therefore ignore the first line of nodes_db_ipv4].
		if [ ! "${node_ipv4}" = "${public_ipv4}" ]
		then
			sed -i "/5432 public_ipv4-var/a \    server postgresql_${node_ipv4}_5432 ${node_ipv4}:5432 maxconn 100 check check-ssl verify none port 8008" "/etc/haproxy/haproxy.cfg"
		fi
	done
	unset node_ipv4

	sed -i "s|public_ipv4-var|${public_ipv4}|g" "/etc/haproxy/haproxy.cfg"

	############################################################################

	dna-echo_operation -h "restarting haproxy on node: | ${l-}${hostname}${n-} |"

	systemctl restart haproxy

	dna-wait -s "4" -d "Now allowing haproxy to start up correctly..."

	dna-echo_operation -t

	############################################################################

}

aenigma-install_haproxy() {

	############################################################################

	### Install dependencies required for postgresql cluster

	aenigma-install_postgresql_dependencies

	############################################################################

	### Install haproxy

	dna-install_dependencies -p "haproxy"

	############################################################################

	### Apply haproxy config

	aenigma-apply_haproxy_config

	############################################################################

}

aenigma-provision_onion_tls_certificate() {

	### Provision self_signed wildcard certificate for $tor_hidden_service_hostname

	dna-provision_self_signed_wildcard_cert -d "${tor_hidden_service_hostname}"

	### Set permissions on for the new certs in $tls_project_dir

	aenigma-set_permissions

	### developing

}

aenigma-provision_tor() {

	### Make server reachable via tor

	############################################################################
	############################################################################

	### Get hostname

	dna-get_hostname

	############################################################################
	############################################################################

	### Install tor

	dna-install_tor

	dna-echo_operation -h "enabling Tor Hidden Service"

	dna-tor_enable_hidden_service

	sed -i '/############### This section is just for location-hidden services ###/,/################ This section is just for relays #####################/{//!d}' "/etc/tor/torrc"
	sed -i "/############### This section is just for location-hidden services ###/ r ${source_dir}/conf/tor/hidden_service" "/etc/tor/torrc"

	sed -i "s|public_ipv4-var|${public_ipv4}|g" "/etc/tor/torrc"

	dna-echo_operation -t

	dna-echo_operation -h "restarting Tor"

	service tor restart

	dna-echo_operation -t

	############################################################################
	############################################################################

	### TLS

	############################################################################

	### Provision self_signed wildcard certificate for $tor_hidden_service_hostname

	if [ ! -f "${tls_project_dir}/${tor_hidden_service_hostname}.cert.pem" ] || [ ! -f "${tls_project_dir}/${tor_hidden_service_hostname}.privkey.pem" ]
	then

		aenigma-provision_onion_tls_certificate

	fi

	############################################################################
	############################################################################

	### NGINX

	############################################################################

	### Install conf for onion hostname

	dna-echo_operation -h "applying NGINX config for Tor"

	cp "${source_dir}/conf/nginx/onion" "/etc/nginx/conf.d/onion.conf"

	sed -i "s/tor_hidden_service_hostname-var/${tor_hidden_service_hostname}/g" "/etc/nginx/conf.d/onion.conf"

	sed -i "s/hostname.xyz/${hostname}/g" "/etc/nginx/conf.d/onion.conf"
	sed -i "s/domain.xyz/${domain}/g" "/etc/nginx/conf.d/onion.conf"

	dna-echo_operation -t

	### Make onion the default_server in nginx to avoid domain leaks in case of domains mismatches

	dna-echo_operation -h "applying NGINX config for Tor"

	sed -i "s|443 default_server ssl|443 ssl|g" "/etc/nginx/conf.d/${domain}.conf"
	sed -i "s|443 ssl|443 default_server ssl|g" "/etc/nginx/conf.d/onion.conf"

	dna-echo_operation -t

	### Restart nginx

	dna-echo_operation -h "restarting NGINX"

	service nginx restart

	dna-echo_operation -t

	############################################################################
	############################################################################

	### Allow .onion JID registrations and federate with other XMPP servers hosted inside the Tor network on hidden services via Tor itself [https://www.grepular.com/Transparent_Access_to_Tor_Hidden_Services]

	dna-echo_operation -h "adding .onion hostname to ejabberd hosts"

	if ! grep -q "${tor_hidden_service_hostname}" "/opt/ejabberd/conf/ejabberd.yml"
	then
		sed -i "/^hosts:$/a \  - \"${tor_hidden_service_hostname}\"" "/opt/ejabberd/conf/ejabberd.yml"
	fi

	dna-echo_operation -t

	dna-echo_operation -h "adding .onion hostname to SSLH config"

	if ! grep -q "xh${node_number}.${tor_hidden_service_hostname}" "/etc/sslh.cfg"
	then
		sed -i "/xmpp-client/a \    { name: \"tls\"; host: \"localhost\"; port: \"5443\"; sni_hostnames: [ \"xh${node_number}.$tor_hidden_service_hostname\" ]; log_level: 1; }," "/etc/sslh.cfg"
	fi

	if ! grep -q "ae${node_number}.${tor_hidden_service_hostname}" "/etc/sslh.cfg"
	then
		sed -i "/spdy/a \    { name: \"tls\"; host: \"localhost\"; port: \"5223\"; sni_hostnames: [ \"ae${node_number}.$tor_hidden_service_hostname\" ]; log_level: 1; }," "/etc/sslh.cfg"
	fi

	dna-echo_operation -t

	dna-echo_operation -h "restarting SSLH"

	service sslh restart

	dna-echo_operation -t

	dna-echo_operation -h "restarting ejabberd"

	service ejabberd restart

	dna-echo_operation -t

	dna-echo_operation -h "enabling Tor DNS resolution for .onion domains"

	### Tor

	if ! grep -q "DNSPort   0.0.0.0:5300" "/etc/tor/torrc"
    then
        cat <<EOF >> "/etc/tor/torrc"
DNSPort   0.0.0.0:5300
VirtualAddrNetworkIPv4 10.192.0.0/11
AutomapHostsOnResolve 1

TransPort 9040
EOF
    fi

	service tor restart

	dna-echo_operation -t

	### Unbound

	dna-echo_operation -h "installing unbound local DNS server"

	dna-install_dependencies -p "unbound"

	cat <<EOF > "/etc/unbound/unbound.conf.d/tor.conf"
server:

domain-insecure: "onion"
private-domain: "onion"
do-not-query-localhost: no
local-zone: "onion." nodefault

forward-zone:
    name: "onion"
    forward-addr: 127.0.0.1@5300
EOF

	service unbound restart

	dna-echo_operation -t

	### Enable unbound as the system-wide dns resolver.

	echo "127.0.0.1" > "/etc/resolv.conf"

	### UFW / IPTABLES

	dna-echo_operation -h "configuring IPtables for Tor routing"

	if ! grep -q "DNSPort   0.0.0.0:5300" "/etc/ufw/before.rules"
	then
		cat <<EOF >> "/etc/ufw/before.rules"
*nat
-A INPUT      -p tcp --dport 9040     -j ACCEPT
-A PREROUTING -p tcp -d 10.192.0.0/11 -j REDIRECT --to-port 9040
-A OUTPUT     -p tcp -d 10.192.0.0/11 -j REDIRECT --to-port 9040
COMMIT
EOF
	fi

	service ufw restart

	sleep 1

	ufw disable && ufw --force enable

	echo

	dna-echo_operation -t

	dna-echo -s "Tor successfully provisioned!"

	############################################################################
	############################################################################

	### Create alternate webroot for onion specific customizations and converse

	############################################################################

	### Create web locations and populate it

	mkdir -p "/var/www/onion/"
	chown -R "www-data:www-data" "/var/www/onion/"

	cp "${source_dir}/conf/web/global/index.html" "/var/www/onion/"

	############################################################################
	############################################################################

}

aenigma-provision_conversejs() {

	### Provision conversejs

	############################################################################

	### Install dependencies

	dna-install_nodejs

	############################################################################

	### Clone and build

	if [ -d "/var/www/${project_name}/xmpp/" ]
	then
		if [ -d "/var/www/${project_name}/xmpp-old/" ]
		then
			rm -rf "/var/www/${project_name}/xmpp-old/"
		fi
		mv "/var/www/${project_name}/xmpp/" "/var/www/${project_name}/xmpp-old/"
	fi

	git clone https://github.com/conversejs/converse.js "/var/www/${project_name}/xmpp/"

	( cd "/var/www/${project_name}/xmpp/" && git checkout "tags/${hardcoded_package_version_conversejs}" && make dev && make dist )

	echo

	### Install libsignal dependency

	( cd "/var/www/${project_name}/xmpp/" && mkdir -p "3rdparty" && curl "https://raw.githubusercontent.com/signalapp/libsignal-protocol-javascript/master/dist/libsignal-protocol.js" > "./3rdparty/libsignal-protocol.js" )

	echo

	### Move fullscreen.html to index.html

	mv "/var/www/${project_name}/xmpp/fullscreen.html" "/var/www/${project_name}/xmpp/index.html"

	### Localize sources

	sed -i "s|https://cdn.conversejs.org/4.2.0/css/converse.min.css|dist/converse.min.css|g" "/var/www/${project_name}/xmpp/index.html"
	sed -i "s|https://cdn.conversejs.org/3rdparty/libsignal-protocol.min.js|3rdparty/libsignal-protocol.js|g" "/var/www/${project_name}/xmpp/index.html"
	sed -i "s|https://cdn.conversejs.org/4.2.0/dist/converse.min.js|dist/converse.min.js|g" "/var/www/${project_name}/xmpp/index.html"

	### Remove analytics

	sed -i "/inverse-analytics.js/d" "/var/www/${project_name}/xmpp/index.html"
	sed -i "/stats.opkode.com\/piwik.php/d" "/var/www/${project_name}/xmpp/index.html"

	### Use our own ejabberd websocket service

	sed -i "s|bosh_service_url|websocket_url|g" "/var/www/${project_name}/xmpp/index.html"
	sed -i "s|https://conversejs.org/http-bind/|wss://xh${node_number}.${domain}/ws/|g" "/var/www/${project_name}/xmpp/index.html"

	### Remove test bosh service warning

	sed -i "s|, // Please use this connection manager only for testing purposes|,|g" "/var/www/${project_name}/xmpp/index.html"

	### Add config parameters

	sed -i "s|view_mode: 'fullscreen'|view_mode: 'fullscreen',|g" "/var/www/${project_name}/xmpp/index.html"

	sed -i "/view_mode/ r /dev/stdin" "/var/www/${project_name}/xmpp/index.html" <<'EOF'
        allow_chat_pending_contacts: true,
        allow_non_roster_messaging: true,
        allow_otr: false,
        allow_registration: true,
        auto_join_on_invite: true,
        csi_waiting_time: 60,
        default_domain: 'domain-var',
        enable_smacks: true,
        muc_show_join_leave: false,
EOF

	sed -i "s|domain-var|${domain}|g" "/var/www/${project_name}/xmpp/index.html"

	### Set permissions

	chown -R www-data:www-data "/var/www/${project_name}/xmpp/"

	############################################################################

	### Also make Converse available over Tor in its alternate webroot

	rsync -aAXXz "/var/www/${project_name}/xmpp/" "/var/www/onion/xmpp/"

	### Use onion-specific websocket

	sed -i "s|wss://xh${node_number}.${domain}/ws/|wss://${tor_hidden_service_hostname}/ws/|g" "/var/www/onion/xmpp/index.html"

	### Use onion domain for default_domain

	sed -i "s|${domain}|${tor_hidden_service_hostname}|g" "/var/www/onion/xmpp/index.html"

	############################################################################
	############################################################################

}

aenigma-set_loglevel_zero_enabled() {

	### ########################################################################

	### turn all logging off globally to prevent backward data leaks should the
	### server be compromised or seized.

	### ########################################################################

	### evaluate arguments.

	### ########################################################################

	while getopts ":e" arguments
	do
		case "${arguments}" in
			e)	skip_ejabberd_stop_start_operations="y"
				;;
			\?)	dna-echo -e "Invalid option | ${n-}-$OPTARG${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
				dna-exit
				;;
		esac
	done
	OPTIND=1

	### ########################################################################

	dna-echo_operation -h "disabling logging for all services"

	### ejabberd.

	### ########################################################################

	sed -i "s|^loglevel.*|loglevel: 0|g" "/opt/ejabberd/conf/ejabberd.yml"

	if [ ! -z "$(ls /opt/ejabberd/logs/* 2> /dev/null)" ]
	then
		rm /opt/ejabberd/logs/*
	fi

	#### restart ejabberd to take effect of the changes.

	if [ -z "${skip_ejabberd_stop_start_operations-}" ]
	then

		aenigma-restart_ejabberd_if_running

	fi

	### ########################################################################

	### sslh.

	### ########################################################################

	sed -i "s|log_level: 1;|log_level: 0;|g" "/etc/sslh.cfg"

	service sslh restart

	### ########################################################################

	### nginx.

	### ########################################################################

	sed -i "s|^\    access_log.*|\    access_log off;|g" "/etc/nginx/nginx.conf"
	sed -i "s|^error_log.*|error_log off;|g" "/etc/nginx/nginx.conf"

	sed -i "s|^\    access_log.*|\    access_log off;|g" /etc/nginx/conf.d/*
	sed -i "s|^\    error_log.*|\    error_log off;|g" /etc/nginx/conf.d/*

	if [ ! -z "$(ls /var/log/nginx/* 2> /dev/null)" ]
	then
		rm /var/log/nginx/*
	fi

	service nginx restart

	### ########################################################################

	### ufw

	ufw logging off
	echo

	### ########################################################################

	dna-echo_operation -t

	unset skip_ejabberd_stop_start_operations

}

aenigma-set_loglevel_zero_disabled() {

	### ########################################################################

	### turn all logging back on if set_loglevel_zero_enabled was previously
	### ran.

	### ########################################################################

	### evaluate arguments.

	### ########################################################################

	while getopts ":e" arguments
	do
		case "${arguments}" in
			e)	export skip_ejabberd_stop_start_operations="y"
				;;
			\?)	dna-echo -e "Invalid option | ${n-}-$OPTARG${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
				dna-exit
				;;
		esac
	done
	OPTIND=1

	### ########################################################################

	dna-echo_operation -h "[re-]enabling logging for all services"

	### ejabberd.

	### ########################################################################

	sed -i "s|^loglevel.*|loglevel: 4|g" "/opt/ejabberd/conf/ejabberd.yml"

	#### restart ejabberd to take effect of the changes.

	if [ -z "${skip_ejabberd_stop_start_operations-}" ]
	then

		aenigma-restart_ejabberd_if_running

	fi

	### ########################################################################

	### sslh.

	### ########################################################################

	sed -i "s|log_level: 0;|log_level: 1;|g" "/etc/sslh.cfg"

	service sslh restart

	### ########################################################################

	### nginx.

	############################################################################

	sed -i "s|\    access_log off;|\    access_log  /var/log/nginx/access.log  main;|g" "/etc/nginx/nginx.conf"
	sed -i "s|error_log off;|error_log  /var/log/nginx/error.log warn;|g" "/etc/nginx/nginx.conf"

	for nginx_config_file in /etc/nginx/conf.d/*.conf
	do
        nginx_site_name="$(echo ${nginx_config_file} | cut -d "/" -f5 | sed -e "s/.conf$//")"
        sed -i "s|\    access_log off;|\    access_log /var/log/nginx/${nginx_site_name}.access.log;|g" "${nginx_config_file}"
	done

	for nginx_config_file in /etc/nginx/conf.d/*.conf
	do
        nginx_site_name="$(echo ${nginx_config_file} | cut -d "/" -f5 | sed -e "s/.conf$//")"
        sed -i "s|\    error_log off;|\    error_log /var/log/nginx/${nginx_site_name}.error.log;|g" "${nginx_config_file}"
	done

	unset nginx_config_file nginx_site_name

	service nginx restart

	### ########################################################################

	### ufw

	ufw logging low
	echo

	### ########################################################################

	dna-echo_operation -t

	unset skip_ejabberd_stop_start_operations

}

aenigma-set_mam_off_enabled() {

	### ########################################################################

	### turn all ejabberd message archiving [mod_mam] off for extra stealth.

	### ########################################################################

	### evaluate arguments.

	### ########################################################################

	while getopts ":e" arguments
	do
		case "${arguments}" in
			e)	skip_ejabberd_stop_start_operations="y"
				;;
			\?)	dna-echo -e "Invalid option | ${n-}-$OPTARG${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
				dna-exit
				;;
		esac
	done
	OPTIND=1

	### ########################################################################

	dna-echo_operation -h "disabling ejabberd message archiving [mod_mam] globally"

	### disable mod_mam in ejabberd.yml.

	sed -i '/^\  mod_mam/,/^\    cache_life_time/{//!d}' "/opt/ejabberd/conf/ejabberd.yml"
	sed -i "s|^\  mod_mam:|#  mod_mam: {}|g" "/opt/ejabberd/conf/ejabberd.yml"
	sed -i "/^\    cache_life_time/d" "/opt/ejabberd/conf/ejabberd.yml"

	### disable mod_mix as it depends on mod_mam.

	sed -i "s|^\  mod_mix:|#  mod_mix: {}|g" "/opt/ejabberd/conf/ejabberd.yml"
	sed -i "/^\    host: \"xm/d" "/opt/ejabberd/conf/ejabberd.yml"

	### delete existing mam archive.

	ejabberdctl --no-timeout delete_old_mam_messages all 0

	### restart ejabberd to take effect of the changes.

	if [ -z "${skip_ejabberd_stop_start_operations-}" ]
	then

		aenigma-restart_ejabberd_if_running

	fi

	dna-echo_operation -t

	unset skip_ejabberd_stop_start_operations

}

aenigma-set_mam_off_disabled() {

	### ########################################################################

	### turn all ejabberd message archiving [mod_mam] back on.

	### ########################################################################

	### evaluate arguments.

	### ########################################################################

	while getopts ":e" arguments
	do
		case "${arguments}" in
			e)	export skip_ejabberd_stop_start_operations="y"
				;;
			\?)	dna-echo -e "Invalid option | ${n-}-$OPTARG${r-} | for function | ${n-}${FUNCNAME[0]}${r-} |."
				dna-exit
				;;
		esac
	done
	OPTIND=1

	### ########################################################################

	dna-echo_operation -h "[re-]enabling ejabberd message archiving [mod_mam] globally"

	aenigma-install_ejabberd_config
	aenigma-localize_ejabberd_install

	dna-echo_operation -t

	unset skip_ejabberd_stop_start_operations

}

### ############################################################################

### 3 | synthia installation.

### ############################################################################

synthia-perform_installation() {

	### display aenigma logo.

	aenigma-display_logo

	### link tools in $path.

	dna-link_tools

	### display intro.

	if [ -z "${skip_intro-}" ]
	then
	    aenigma-xmpp_intro
	fi

	### check for distro compatibility.

	if ! dna-check_distro_ubuntu_18 && ! dna-check_distro_debian_9
	then
		dna-echo -e "${project_name} only runs on Ubuntu 18.04 LTS [bionic] or Debian 9 [stretch]."
		dna-echo -m "This is due - among other factors - to the fact that the ejabberd version installed by our APT repo of choice on previous releases is too old."
		dna-echo -m "You can edit out this check to proceed at your own risk."
		dna-exit
	fi

	### abort installation if this session is open via sslh as we'll have to
	### stop sslh itself during the course of this installation.

	dna-abort_on_sslh_session

	### run inception skipping anything that aenigma will set later on anyway.

	dna-inception -h -m -p

	### refresh tmp_project_dir.

	dna-refresh_tmp_project_dir

	### create tls_project_dir.

	mkdir -p "${tls_project_dir}"

	### globally allow APT to fail during the course of the installation until
	### the final check round.

	export dna_global_allow_apt_errors="y"

	### legacy fixes and migration detections.

	### ########################################################################

	#### dhparam.

	if [ -f "${tls_project_dir}/dh.pem" ]
	then
		rm "${tls_project_dir}/dh.pem"
	fi

	#### ejabbed sql.

	if dpkg -s ejabberd &> /dev/null
	then
		if [ -f "/etc/ejabberd/ejabberd.yml" ]
		then
			if ! grep -q "default_db: sql" "/etc/ejabberd/ejabberd.yml"
			then
				export psql_migration_required="true"
			fi
		elif [ -f "/opt/ejabberd/conf/ejabberd.yml" ]
		then
			if ! grep -q "default_db: sql" "/opt/ejabberd/conf/ejabberd.yml"
			then
				export psql_migration_required="true"
			fi
		fi
	fi

	#### nginx uploads.

	if [ -f "/etc/nginx/conf.d/aenigma.conf" ]
	then
		rm "/etc/nginx/conf.d/aenigma.conf"
	fi

	if [ -f "/etc/apt/sources.list.d/nginx.list" ]
	then

		rm /etc/apt/sources.list.d/nginx.list*
		apt-get update

		dna-check_dpkg_availability
		export DEBIAN_FRONTEND=noninteractive ; apt-get install -y -o Dpkg::Options::="--force-confnew" --force-yes nginx-common || true
		apt-get -y install nginx || true
		dna-nginx_increase_names_hash_bucket_size_128

		if [ -f "/etc/nginx/conf.d/onion.conf" ]
		then

			if grep -q "default_server" "/etc/nginx/conf.d/onion.conf"
			then

				sed -i "s|443 default_server ssl|443 ssl|g" "/etc/nginx/conf.d/${domain}.conf"

			fi

		fi

		if [ -d "/etc/nginx/sites-enabled/" ]
		then
			rm -rf "/etc/nginx/sites-enabled"
		fi

		if [ -f "/etc/nginx/conf.d/default.conf" ]
		then
			rm "/etc/nginx/conf.d/default.conf"
		fi

		service nginx restart

	fi

	#### ejabberd from apt to deb.

	if [ -f "/etc/apt/sources.list.d/jabber.at.list" ]
	then

		rm "/etc/apt/sources.list.d/jabber.at.list"

		apt-get update
		echo

		service ejabberd stop || true
		systemctl disable --now ejabberd || true
		systemctl daemon-reload || true

		apt-get -y purge ejabberd || true
		echo

		apt-get -y purge erlang-p1-pgsql || true
		echo

		rm /etc/systemd/system/ejabberd* || true
		rm /usr/sbin/ejabberdctl || true

		epmd -kill || true

	fi

	### ########################################################################

	### Ask user if this aenigma node is going to be part of a cluster.

	### ########################################################################

	describe_setting() {

		dna-echo -m "${project_name} can be setup to run in a cluster of multiple servers, all working together to make your instance run smoothly, robustly, and be fault-tolerant and load-balancing."
		dna-echo -m "If you wish to setup ${project_name} in a cluster configuration, you must specify so now."

	}

	validate_input() {

		case $input in

			single)		valid="y"
						;;
			cluster)	valid="y"
						;;
			*)			dna-echo -e "Invalid setting for | ${n-}cluster_enabled${r-} | in function | ${n-}${FUNCNAME[0]}${r-} |."
						valid="n"
						;;

		esac

	}

	dna-set_option -o "cluster_enabled" -p "cluster" -d "y" -c "[single/cluster]" -v "y"

	if [ "${cluster_enabled}" = "cluster" ]
	then

		describe_setting() {

			dna-echo -m "Having chosen to run ${project_name} in a cluster of servers, you must now designate your master node that will be performing some special operations to lead the cluster."

		}

		validate_input() {

			case $input in

				primary)	valid="y"
							;;
				secondary)	valid="y"
							;;
				*)			dna-echo -e "Invalid setting for | ${n-}cluster_enabled${r-} | in function | ${n-}${FUNCNAME[0]}${r-} |."
							valid="n"
							;;

			esac

		}

		dna-set_option -o "cluster_mode" -p "cluster" -d "y" -c "[primary/secondary]" -v "y"

		if [ "${cluster_mode-null}" = "primary" ]
		then

			dna-echo -m "Having chosen that this server will be your primary node for your ${project_name} installation, we can carry on with the standard installation."

		else

			dna-echo -m "Having chosen that this server will be a secondary node for your ${project_name} installation, we first need to make sure that there is an already running primary node that is already up and running and perfectly configured."
            dna-ask_for_boolean -d "y" -q "Is the primary node for this ${project_name} already up and running and perfectly configured?"

            if [ "${boolean_output}" = "y" ]
            then

				dna-echo -m "Ok, let's then continue on with the configuration of this secondary node and proceed to have it join the cluster lead by your already running primary node..."

            else

				dna-echo -m "Ok, no problem. Please proceed with the full installation and configuration of your primary node on its respective server by following the same ${project_name} install and selecting the | ${y-}primary${n-} | node option when prompted."
				dna-echo -m "Only then will we be able to continue with the installation of this secondary node."
				dna-echo -m "Aborting installation for now."
                dna-exit

            fi

		fi

	fi

	### ########################################################################

	### perform primary_node-only setup.

	### ########################################################################

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		#### start creating the nodes_db files and print this machine [the
		#### primary node]'s public IPv4 and IPv6 [if available] to the first
		#### line of each.
		#### we create the nodes_db file independently of whether clusterization
		#### is enabled or not.

		if [ "${cluster_enabled}" = "single" ]
		then

			##### clusterization has been turned off.
			##### delete the conf_nodes directory if present.

			if [ -d "${conf_cluster_nodes_dir}" ]
			then

				rm -rf "${conf_cluster_nodes_dir}"

			fi

		fi

		#### set xmpp domain.

	    describe_setting() {

	        dna-echo -m "Now specify your top level domain, which will become the part after the @ in your JIDs [XMPP account addresses]."
			dna-echo -m "In the next step you'll be able to specify additional domains for which all XMPP services will be fully provided just like this one."
			dna-echo -m "The domain you'll be specifying right now will simply be the administrative domain for this and any other machines part of this ${project_name} instance."

	    }

		validate_input() {

			dna-validate_domain -std "${input}"

			if [ "${domain_is_valid}" = "y" ]
			then
				valid="y"
			else
				dna-echo -m "Check the syntax to ensure it is in fact a top-level domain and try again."
				valid=n
			fi

			unset silent

		}

	    dna-set_option -o "domain" -p "xmpp" -d "y" -v "y"

	    if [ "${previous_value_outcome}" = "not-kept" ]
	    then

	        dna-echo -m "- * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! -"
	        dna-echo -e "WARNING:"
	        dna-echo -e "Changing domains will not preserve any user accounts or data. It will revert this ejabberd instance to a fresh install."
	        dna-echo -m "In the future, all user accounts and data for the previous domain might be recovered automatically if you ever re-run this install and select the old domain, but this hasn't been thoroughly tested yet."
	        dna-echo -m "If you answer no to the following question, nothing will be deleted."
	        dna-echo -m "- * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! -"

            dna-ask_for_boolean -d "n" -q "Are you absolutely sure you want to change the XMPP domain now?"
            if [ "${boolean_output}" = "y" ]
            then
	            dna-echo -m "Ok, proceeding..."
	        else
	            dna-echo -m "Ok, setting your domain back to the previous value | ${y-}${previous_value}${n-} |..."
	            domain="${previous_value}"
	            echo "${previous_value}" > "${opt_path}"
	        fi

	    fi

		#### set additional xmpp domains.

	    describe_setting() {

	        dna-echo -m "Now specify any additional domains, for which XMPP services will be provided alongside your main domain."
			dna-echo -m "Domains must be inputted in space separated format, such as: | ${y-}abc.com xyz.org 123.net${n-} |."
			dna-echo -m "If you don't wish to specify any additional domains, simply type: | ${y-}none${n-} |."

	    }

		validate_input() {

			if [ "${input}" = "none" ]
			then

				valid="y"

			else

				additional_domains_array=(${input})
				for additional_domain in "${additional_domains_array[@]}"
			    do

					dna-validate_domain -std "${additional_domain}"

					if [ "${domain_is_valid}" = "y" ]
					then
						valid="y"
					else
						dna-echo -m "Check the syntax to ensure it is in fact a top-level domain and try again."
						valid=n
						break
					fi

				done

				unset silent

			fi

		}

	    dna-set_option -o "additional_domains" -p "xmpp" -d "y" -v "y"

	    if [ "${previous_value_outcome}" = "not-kept" ]
	    then

	        dna-echo -m "- * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! -"
	        dna-echo -e "WARNING:"
	        dna-echo -e "If you are removing any additional domains, this operation will not preserve any user accounts or data related to such domains."
	        dna-echo -m "In the future, all user accounts and data for the previous domains might be recovered automatically if you ever re-run this install and specify the same additional domains as before, but this hasn't been thoroughly tested yet."
	        dna-echo -m "If you answer no to the following question, nothing will be deleted."
			dna-echo -m "This only applies to removed domains and you can ignore this warning entirely if you are NOT removing any domain [but only adding]."
	        dna-echo -m "- * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! - * - !!! -"

            dna-ask_for_boolean -d "n" -q "Are you absolutely sure you want to change your additional XMPP domains now?"
            if [ "${boolean_output}" = "y" ]
            then
	            dna-echo -m "Ok, proceeding..."
	        else
	            dna-echo -m "Ok, setting your additional domains back to the previous value | ${y-}${previous_value}${n-} |..."
	            additional_domains="${previous_value}"
	            echo "${previous_value}" > "${opt_path}"
	        fi

	    fi

		#### review and set machine hostname.

		curr_host_name="$(cat /etc/hostname)"

	    if [ "${curr_host_name}" = "ae01.${domain}" ]
	    then

			dna-echo -s "Your hostname is already correctly set to: | ${n-}ae01.${domain}${g-} |."

		else

			dna-echo -m "This machine's hostname must become: | ${y-}ae01.${domain}${n-} |."
			dna-echo -m "This allows for a correct setup of ejabberd that will make it possible in the future to turn this instance of ${project_name} into a cluster should you ever choose to do so."
			dna-echo -m "Your current hostname is:"
			dna-echo -m " | ${y-}${curr_host_name}${n-} | "

            dna-ask_for_boolean -d "y" -q "Do you want to correctly set your hostname automatically?"

            if [ "${boolean_output}" = "y" ]
            then

			    dna-echo -m "Ok, setting hostname to: | ${y-}ae01.${domain}${n-} |."
			    echo "ae01.${domain}" > "/etc/hostname"

			else

			    dna-echo -m "Ok, not changing hostname."
				dna-exit

			fi

	    fi

		dna-get_hostname

		hostname -F "/etc/hostname"

		#### review domain and hostname to see if the user approves.

		dna-echo -m "----------------------------------------------------------------"

		dna-echo -m "To make sure everything is correct:"

		dna-echo -m "----------------------------------------------------------------"

		dna-echo -m "1] Your primary | ${o-}domain${n-} | [the part after the @ in your XMPP account addresses] will be:"
		dna-echo -m " | ${y-}${domain}${n-} | "
		dna-echo -m "2] And therefore your | ${o-}JIDs [XMPP account addresses]${n-} | will look as follows:"
		dna-echo -m " | ${y-}aurora@${domain}${n-} | "
		dna-echo -m "3] Your | ${o-}hostname${n-} | [the location on the internet of this server] will be:"
		dna-echo -m " | ${y-}${hostname}${n-} | "
		dna-echo -m "4] And therefore your | ${o-}${project_name} admin panel${n-} | will be located at:"
		dna-echo -m " | ${y-}https://${hostname}${n-} | "

		dna-echo -m "----------------------------------------------------------------"

		if [ ! "${additional_domains-null}" = "none" ]
		then

			dna-echo -m "5] In addition, your | ${o-}additional domains${n-} | will be:"
			dna-echo -m " | ${y-}${additional_domains}${n-} | "

			dna-echo -m "----------------------------------------------------------------"

		fi

		dna-ask_for_boolean -d "y" -q "Does everything look all right?"

		if [ "${boolean_output}" = "y" ]
		then

		    dna-echo -m "Ok, continuing."

		else

		    dna-echo -m "Ok, no worries. You can re-run this script right now and make the correct choices. Exiting..."
		    dna-exit

		fi

		#### set node_number to 01 as this is the primary node.


		dna-set_option -o "node_number" -p "cluster" -s "01"

		#### initialize nodes_db files.

		aenigma-initialize_nodes_db_files

		#### update routing info.

		aenigma-update_routing_info

	fi

	### set admin email address.

	dna-set_admin_email

	### check internet connectivity and get public ip addresses.

	dna-get_public_ipv4

	dna-check_ipv6_connectivity

	### create the necessary files and directories for ejabberd.

	### ########################################################################

	#### ejabberd locations.

	aenigma-create_ejabberd_locations

	#### $conf_xmpp_dir.

	mkdir -p "${conf_xmpp_dir}"

	### ########################################################################

	### add cronjob for resync_cluster script.

	if [ "${cluster_enabled}" = "cluster" ]
	then

		dna-add_cron_job -n "${project_name}-resync_cluster" -t "@reboot" -c "bash /root/openspace42/${project_name}/tools/${project_name}-resync_cluster --return-check"

	fi

	### proceed with primary/secondary_node-split installation.

	### ########################################################################

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		#### create domains_db.

		aenigma-create_domains_db

		#### perform dns checks.

		aenigma-perform_dns_checks

		#### install certbot preemtively and place the post renewal hook.

		#### ###################################################################

		##### install certbot.

		dna-install_certbot

		##### copy the LetsEncrypt post-renewal script to its location and
		##### actually run it now to be ready for the installation.

		cp "${source_dir}/conf/letsencrypt/post-renewal-hook" "/etc/letsencrypt/renewal-hooks/post/${project_name}-create-push-certs"

		##### execute create_push_certs now to generate the all-in-one tls cert
		##### for ejabberd.

		aenigma-create_push_certs -e

		#### ###################################################################

		#### setup backup restore.

		dna-setup_backup_restore

	else

		#### clear any letsencrypt post-renewal-hook if present from a previous
		#### installation in primary node mode.

		if [ -f "/etc/letsencrypt/renewal-hooks/post/${project_name}-create-push-certs" ]
		then
			rm "/etc/letsencrypt/renewal-hooks/post/${project_name}-create-push-certs"
		fi

		#### generate token [or use the public IPv6 address of this machine if
		#### available] as visual fingerprint for user connection checking.

		if [ "${ipv6_available}" = "y" ]
		then

			echo "${this_ipv6}" > "${tmp_project_dir}/aenigma-cluster-token"

		else

			aenigma_cluster_token="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 24 | head -n 1)"
			echo "${aenigma_cluster_token}" > "${tmp_project_dir}/aenigma-cluster-token"

		fi

		#### temporarily allow incoming connections on port 42022 without limit
		#### on UFW to avoid locking out the primary node during installation.

		/usr/sbin/ufw allow 42022
		echo

		#### echo public ipv4 of this machine for clusterization.

		dna-echo -m "This is the public IPv4 for this machine:"
		dna-echo -m " | ${y-}$public_ipv4${n-} | "
		dna-echo -m "Enter it on the primary node when prompted to do so, then follow the instructions from there."

		sleep 1

		#### create ssh authorized_keys file.

		if [ -f "${ssh_auth_keys_file}" ]
		then

	        dna-echo -s "SSH Authorized Keys file found."

		else

			dna-echo -m "SSH Authorized Keys file NOT found. Creating it now."
			mkdir -p "/root/.ssh/"
	        touch "${ssh_auth_keys_file}"

		fi

		#### ask for primary node ssh pubkey if not already present.

		if grep -q "ae01." "${ssh_auth_keys_file}"
		then

			dna-echo -s "An SSH public key generated by the primary node was already previously saved on this server."

			check_condition() {

				if [ -f "${tmp_project_dir}/aenigma-cluster-ssh-connection-successful" ]
				then
					condition_verified="y"
				fi

			}

			dna-wait_for -m "32" -d "Now looking to see if the primary node is already able to connect to this node..."

			if [ "${wait_output}" = "ok" ]
			then

				dna-echo -s "Primary node SSH connection to this node succeeded. Continuing installation..."
				primary_node_ssh_connection="ok"
				rm -r "${tmp_project_dir}/aenigma-cluster-ssh-connection-successful"

			else

				dna-echo -e "The primary node was not able to connect to this node with the SSH public key already saved on this machine."
				dna-echo -m "You will now need to specify the SSH public key displayed on your primary node."
				dna-echo -m "[or if it manages to connect in the mean time, simply type: | ${y-}skip${n-} |]"
				primary_node_ssh_connection="attempted"

			fi

			unset wait_output

		fi

		#### check for successful connection from primary node.

		if [ ! "${primary_node_ssh_connection-null}" = "ok" ]
		then

			validate_input() {

				if [ "${input}" = "skip" ]
				then

					if [ -f "${tmp_project_dir}/aenigma-cluster-ssh-connection-successful" ]
					then

						valid="y"

					else

						dna-echo -e "The primary node still does NOT appear to have been able to successfully connect to this node."
						dna-echo -m "Ensure its connection to this machine has actually succeeded and, if not, specify the public SSH key it will display shortly."
						valid=n

					fi

				else

					dna-validate_ssh_pubkey -s -k "${input}"

					if [ "${key_is_valid}" = "y" ]
					then

						valid="y"

					else

						dna-echo -e "The key you specified does NOT appear to be a valid SSH Public Key."
						valid="n"

					fi

					unset silent

				fi

			}

			if [ "${primary_node_ssh_connection-null}" = "attempted" ]
			then
				dna-set_value -d "Now specify the public SSH key [or write | ${y-}skip${n-} | if the connection was already successful]:" -v "y"
			else
				dna-set_value -d "Now specify the public SSH key that will be displayed shortly on the primary node:" -v "y"
			fi

			cluster_ssh_pubkey="${value}"

			if [ ! "${value}" = "skip" ]
			then

				if ! grep -q "${cluster_ssh_pubkey}" "${ssh_auth_keys_file}"
				then
					echo "$cluster_ssh_pubkey" >> "${ssh_auth_keys_file}"
				fi

			fi

			unset value

		fi

		#### print clusterization token and instructions.

		dna-echo -m "A token has been generated to verify a correct connection to this machine from the primary node."
		dna-echo -m "You'll be asked to verify it matches with the one that will shortly be displayed on the primary node."
		dna-echo -m "Its value is: | ${y-}$(cat ${tmp_project_dir}/aenigma-cluster-token)${n-} |."

		dna-echo -m "After that, continue the setup on the primary node and come back here when finished."

		sleep 1

		#### listen for setup-complete token.

		check_condition() {

			if [ -f "${tmp_project_dir}/aenigma-cluster-primary-node-setup-complete" ]
			then
				condition_verified="y"
			fi

		}

		dna-wait_for -d "Now waiting for setup on the primary node to be complete..."

		rm -r "${tmp_project_dir}/aenigma-cluster-primary-node-setup-complete"

		dna-echo -s "Primary node setup complete. Now continuing installation..."

		unset wait_output

		sleep 1

		#### re-limit port 42022 on ufw.

		/usr/sbin/ufw limit 42022
		echo

		#### read newly received config.

		dna-read_conf_settings

		aenigma-set_secondary_node_hostname

		#### install duplicity.

		dna-install_duplicity

	fi

	### initiate actual services installation.

	### ########################################################################

	#### ufw.

	dna-echo_operation -h "setting UFW rules"

	ufw allow 5222
	ufw allow 5223
	ufw allow 5269
	ufw allow 5443
	ufw allow 80
	ufw allow 443

	echo

	dna-echo_operation -t

	#### nginx.

	aenigma-provision_nginx

	#### mail notifications.

	dna-install_mail_notifications -f

	#### sslh.

	aenigma-install_sslh

	#### etcd / haproxy / postgresql / patroni.

	#### #######################################################################

	##### etcd.

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		aenigma-install_etcd_cluster_conditionally

		aenigma-apply_and_restart_patroni_haproxy_ejabberd -r

	fi

	##### haproxy.

	aenigma-install_haproxy

	##### postgresql / patroni.

	aenigma-install_postgresql_patroni

	#### #######################################################################

	#### ejabberd.

	#### #######################################################################

	##### remove the ejabberd_admin_password_set file if it exists as ejabberd
	##### was not currently installed.

	aenigma-get_ejabberd_installed_version

	if [ "${ejabberd_installed_version}" = "none" ]
	then

		if [ -f "${conf_xmpp_dir}/ejabberd_admin_password_set" ]
		then
			rm "${conf_xmpp_dir}/ejabberd_admin_password_set"
		fi

	fi

	##### empty the /var/lib/ejabberd/ directory as we're now migrating to psql.
	##### this is only for pre-v0.7.x to v0.7.0 migrations as v0.7.1 introduces
	##### deb package installation for ejabberd which completely changes directory
	##### structures.

	if [ "${psql_migration_required-null}" = "true" ]
	then

		###### stop ejabberd.

		service ejabberd stop

		###### copy dir to backup location.

		rsync -aAXxz /var/lib/ejabberd/ /var/lib/ejabberd-before-psql-migration

		###### empty the dir.

		rm -r /var/lib/ejabberd/*

		###### restore the uploads dir.

		rsync -aAXxz /var/lib/ejabberd-before-psql-migration/uploads/ /var/lib/ejabberd/uploads/

	fi

	##### perform a full apt upgrade.

	if [ "${apt_upgrade-null}" = "n" ]
	then

		dna-apt_full_upgrade -e

	fi

	##### actually install ejabberd from deb package.

	aenigma-install_ejabberd_deb

	##### install the aenigma ejabberd config.

	aenigma-install_ejabberd_config

	##### localize ejabberd configuration without restarting it.

	aenigma-localize_ejabberd_install -s

	##### enable ejabberd.

	aenigma-enable_ejabberd

	##### start ejabberd.

	aenigma-start_ejabberd

	##### only for psql migration

	if [ "${psql_migration_status-null}" = "complete" ]
	then
		ufw allow 5222
		ufw allow 5223
		ufw allow 5444
		echo
	fi

	##### perform primary_node-only operations on ejabberd.

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		###### add backup script to crontab.

		dna-add_cron_job -n ${project_name}-backup -t "8 4 * * *" -c "bash /root/openspace42/${project_name}/tools/${project_name}-backup"

		###### add create_push_certs script to crontab for monthly LE
		###### autorenewal and distribution to other nodes.

		dna-add_cron_job -n ${project_name}-backup -t "8 4 2 * *" -c "bash /root/openspace42/${project_name}/tools/${project_name}-create_push_certs"

		###### register ejabberd admin user.

		dna-echo -m "Now registering ejabberd admin user [if it doesn't yet exist]..."

		ejab_admin_pw="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 24 | head -n 1)"

		/usr/sbin/ejabberdctl register admin "$domain" "$ejab_admin_pw" &> /dev/null || true

		dna-echo -s "Finished registering ejabberd admin user."

		if [ -f "${conf_xmpp_dir}/ejabberd_admin_password_set" ]
		then

            dna-ask_for_boolean -d "y" -q "Leave ejabberd admin user password the same as previously set?"

            if [ "${boolean_output}" = "y" ]
            then

		        dna-echo -m "Ok, skipping ejabberd admin user password reset."

		        ejab_admin_pwreset="n"

		    else

		        dna-echo -m "Ok, resetting ejabberd admin user password to: | ${y-}${ejab_admin_pw}${n-} |."

		        /usr/sbin/ejabberdctl change_password admin "${domain}" "${ejab_admin_pw}"

		        ejab_admin_pwreset="y"

		    fi

		else

		    touch "${conf_xmpp_dir}/ejabberd_admin_password_set"

		    ejab_admin_pwreset="y"

		fi

		###### update ejabberd's module repo.

		dna-echo -m "Now updating ejabberd's module repo..."

		set +e
		/usr/sbin/ejabberdctl modules_update_specs
		set -e

		dna-echo -s "Finished updating ejabberd's module repo..."

		###### create shared roster groups.

		dna-echo -m "Now creating 'ADMIN:EVERYBODY' and 'ADMIN:ONLINE' shared roster groups [admin@${domain} can see 'everybody' and 'all online']..."

		/usr/sbin/ejabberdctl srg_create everybody "$domain" "everybody [$domain]" "This aenigma group includes every user on this domain" ""
		/usr/sbin/ejabberdctl srg-user-add @all@ localhost everybody "$domain"
		/usr/sbin/ejabberdctl srg_create online "$domain" "online [$domain]" "This aenigma group includes every user who is online on this domain" ""
		/usr/sbin/ejabberdctl srg-user-add @online@ localhost online "$domain"
		/usr/sbin/ejabberdctl srg_create admin "$domain" admin "This is a shared roster group set by aenigma" "everybody\\nonline"
		/usr/sbin/ejabberdctl srg-user-add admin "$domain" admin "$domain"

		dna-echo -s "Finished creating shared roster groups."

	fi

	#### #######################################################################

	#### run some generic operations to catch services status output and alert
	#### the user if something has gone wrong.

	declare -A service_checker_array=(
			[APT]="apt-get -y upgrade"
			[NGINX]="service nginx restart"
			[SSLH]="service sslh restart"
		)

	add_service_check_bottom_spacing() {

		if [ "${service}" = "APT" ] || [ "${service}" = "NGINX" ]
		then
			echo
		fi

	}

	for service in "${!service_checker_array[@]}"
	do

		set +e

		if ${service_checker_array[$service]}
		then

			add_service_check_bottom_spacing

			if [ "${service}" = "SSLH" ]
			then
				dna-echo -ns "All | ${n-}${service}${g-} | operations completed successfully."
			else
				dna-echo -s "All | ${n-}${service}${g-} | operations completed successfully."
			fi

		else

			add_service_check_bottom_spacing
			dna-echo -e "An error has occured during | ${n-}${service}${r-} | operations."
			dna-echo -m "This was one of the last steps in this installation, so it's possible that your end result is at least partially working."
			dna-echo -m "Review the output above and try running the installation again."
			dna-press_to_continue

		fi

		set -e

	done

	### ########################################################################

	### if this is the primary node, then echo ejabberd admin credentials and,
	### if cluster is enabled, prompt for clusterization. otherwise, if this is
	### a secondary node, join the primary node.

	if [ ! "${cluster_mode-null}" = "secondary" ]
	then

		#### print login information and credentials.

		dna-echo -s "You can finally log in to your web administration panel:"

		dna-echo -m " | ${y-}https://${hostname}${n-} | "
		dna-echo -m " | ${y-}admin@${domain}${n-} | "

		if [ "${ejab_admin_pwreset}" = "y" ]
		then
		    dna-echo -m " | ${y-}${ejab_admin_pw}${n-} | "
		else
		    dna-echo -m " | ${y-}[your previously set password]${n-} | "
		fi

		dna-press_to_continue

		#### prompt for clusterization.

		if [ "${cluster_enabled-null}" = "cluster" ]
		then

			dna-echo -m "It's now time to run the | ${o-}clusterize${n-} | script to start adding [or updating] your secondary nodes."
            dna-ask_for_boolean -d "y" -q "Are you ready to do so now [definitely proceed if you've just updated]?"

			if [ "${boolean_output}" = "y" ]
            then

				dna-echo -n -m "Ok, running the | ${o-}clusterize${n-} | script now..."

				bash "${tools_dir}/${project_name}-clusterize"

				while [ ! "${exit_clusterization_loop-null}" = "y" ]
				do

					dna-ask_for_boolean -d "y" -q "Would you like to clusterize another node now?"

					if [ "${boolean_output}" = "y" ]
					then

						dna-echo -n -m "Ok, running the | ${o-}clusterize${n-} | script again now..."

						bash "${tools_dir}/${project_name}-clusterize"

					else

						dna-echo -m "Ok, exiting clusterization phase."

						exit_clusterization_loop="y"

					fi

				done

			else

				dna-echo -m "Ok, you can run the | ${o-}clusterize${n-} | script at any time by executing:"
				dna-echo -m " | ${y-}bash ${tools_dir}/${project_name}-clusterize${n-} | "
				dna-press_to_continue

			fi

		fi

	else

		#### join primary node.

		dna-echo -m "Now joining ejabberd cluster by connecting to the primary node..."

		if /usr/sbin/ejabberdctl --no-timeout join_cluster "ejabberd@ae01.${domain}"
		then

			dna-echo -s "The ejabberd cluster was successfully joined!"

		else

			dna-echo -e "The ejabberd cluster join operation failed."
			dna-echo -m "Inspect the | ${y-}/opt/ejabberd/logs/error.log${n-} | logfile for more information."
			dna-exit

		fi

	fi

	### prompt for ejabberd customizations.

	### ########################################################################

	#### prompt for loglevel_zero.

	#### #######################################################################

	describe_setting() {

		dna-echo -m "aenigma can disable all XMPP-related services logging on this server."
		dna-echo -m "This is always a good idea in case the server gets compromised or seized in the future, to at least avoid leaking highly detailed user connection / disconnection and traffic data."
		dna-echo -m "This will apply to ejabberd, SSLH, and NGINX. You can always revert this setting by re-running the installation and setting this option to: | ${y-}disabled${n-} | at this prompt or running the: | ${y-}aenigma-apply_loglevel_standard${n-} | function from | ${o-}aenigma-env${n-} |."
		dna-echo -m "If you'd like to enable | ${l-}loglevel_zero mode${n-} | you must specify so now."

	}

	validate_input() {

		case $input in

			enabled)	valid="y"
						;;
			disabled)	valid="y"
						;;
			*)			dna-echo -e "Invalid setting for | ${n-}loglevel_zero_mode${r-} | in function | ${n-}${FUNCNAME[0]}${r-} |."
						valid="n"
						;;

		esac

	}

	dna-set_option -o "loglevel_zero_mode" -p "stealth" -d "y" -c "[enabled/disabled]" -v "y"

	if [ "${loglevel_zero_mode}" = "enabled" ]
	then
		aenigma-set_loglevel_zero_enabled -e
	else
		aenigma-set_loglevel_zero_disabled -e
	fi

	#### #######################################################################

	#### prompt for mam_off.

	#### #######################################################################

	describe_setting() {

		dna-echo -m "aenigma can disable all ejabberd message archive management [mod_mam] and delete all existing mam archives retroactively."
		dna-echo -m "This can reduce even further the amount of leaked data in the event of this server being compromised or seized in the future [even though OMEMO-encrypted message are always safe anyway]."
		dna-echo -m "You can always revert this setting by re-running the installation and setting this option to: | ${y-}disabled${n-} | at this prompt or running the: | ${y-}aenigma-set_mam_enabled${n-} | function from | ${o-}aenigma-env${n-} |."
		dna-echo -m "If you'd like to enable | ${l-}mam_off mode${n-} | you must specify so now."

	}

	validate_input() {

		case $input in

			enabled)	valid="y"
						;;
			disabled)	valid="y"
						;;
			*)			dna-echo -e "Invalid setting for | ${n-}mam_off_mode${r-} | in function | ${n-}${FUNCNAME[0]}${r-} |."
						valid="n"
						;;

		esac

	}

	dna-set_option -o "mam_off_mode" -p "stealth" -d "y" -c "[enabled/disabled]" -v "y"

	if [ "${mam_off_mode}" = "enabled" ]
	then
		aenigma-set_mam_off_enabled -e
	else
		aenigma-set_mam_off_disabled -e
	fi

	#### #######################################################################

	### ########################################################################

	### prompt for extra services installation.

	### ########################################################################

	#### prompt for tor provisioning.

	#### #######################################################################

	dna-echo -m "aenigma can also provision a Tor hidden service on this machine to make it reachable entirely inside the Tor network."
	dna-echo -m "You will be able to connect to the | ${o-}.onion${n-} | address that will be generated to access all services [XMPP, web, SSH, etc...]"

	dna-ask_for_boolean -d "y" -q "Would you like to provision Tor now?"

	if [ "${boolean_output}" = "y" ]
	then
		aenigma-provision_tor
	fi

	#### #######################################################################

	#### prompt for conversejs provisioning.

	#### #######################################################################

	dna-echo -m "aenigma can also provision a Converse.js instance on this machine to allow you and this instance's users to have your own XMPP web client always available from anywhere."
	dna-echo -m "You will be able to connect to it at | ${o-}https://${hostname}/xmpp/${n-} | from any device."
	dna-echo -m "This operation will take about 5 minutes to complete."

	dna-ask_for_boolean -d "y" -q "Would you like to provision Converse.js now?"

	if [ "${boolean_output}" = "y" ]
	then
		aenigma-provision_conversejs
	fi

	#### #######################################################################

	### ########################################################################

	### generate aenigma-specific dhparam file.

	### ########################################################################

	dna-generate_dhparam

	#### if a dhparam file was found or generated, amend the config file to
	#### include them:

	if [ "${dh_param_status}" = "ok" ]
	then

		if grep -q "### dhfile" "/opt/ejabberd/conf/ejabberd.yml"
		then

			sed -i "s|### dhfile: 'DHFILE'|dhfile: 'DHFILE'|g" "/opt/ejabberd/conf/ejabberd.yml"
			sed -i "s|### s2s_dhfile: 'DHFILE'|s2s_dhfile: 'DHFILE'|g" "/opt/ejabberd/conf/ejabberd.yml"

		fi

	fi

	### ########################################################################

	### restart ejabberd to take effect of the last customizations we applied.

	aenigma-restart_ejabberd_if_running

	### ########################################################################

	### set marker for primary node to read and be informed of this node's
	### installation completion.

	if [ "${cluster_mode-null}" = "secondary" ]
	then

		touch "/tmp/aenigma_secondary_node_provisioning_complete"

	fi

	### delete tmp_project_dir as the installation is complete.

	dna-remove_tmp_project_dir

}

### ############################################################################