organizing initialization sequence

thedatasociety · Jul 27, 2019 · 6fc741b · 6fc741b
1 parent e483204
commit 6fc741b
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 26 deletions.
diff --git a/binder/start b/binder/start
@@ -1,14 +1,22 @@
 #!/bin/bash
 
 
-chmod +x ./binder/start.hadoop 
-chmod +x ./binder/start.hive 
-chmod +x ./binder/start.mysql 
-chmod +x ./binder/start.hbase 
-#chmod +x ./binder/start.mariadb 
+chmod +x ~/binder/start.hadoop 
+chmod +x ~/binder/start.hive 
+chmod +x ~/binder/start.mysql 
+chmod +x ~/binder/start.hbase 
+#chmod +x ~/binder/start.mariadb 
 
+# starting mysql in parallel
+~/binder/start.mysql   &
+
+~/binder/start.hadoop  && \
+sleep 3                && \
+~/binder/start.hive    && \
+sleep 3                && \
+~/binder/start.hbase   && \
+touch ready.txt         &
 
-./binder/start.hadoop && sleep 3 && ./binder/start.hive &&  sleep 3 && ./binder/start.hbase && ./binder/start.mysql  &
 
 
 # DO NOT REMOVE(!) 

diff --git a/binder/start.hadoop b/binder/start.hadoop
@@ -2,33 +2,33 @@
 
 #loading envvars
 #for auto load in binderhub:
-source $(pwd)/.profile
+source ~/.profile
 #for manual start: run 
 #source $(pwd)/.bashrc 
 
 #Starting sshd server
-/usr/sbin/sshd -f $(pwd)/resources/configs/ssh/sshd_config 
+/usr/sbin/sshd -f ~/resources/configs/ssh/sshd_config 
 
 # Adding names to know hosts. This step avoids yes/no host confirmation.
 ssh -o "StrictHostKeyChecking no" $USER@localhost -p 8822 -C "exit" 
 ssh -o "StrictHostKeyChecking no" $USER@0.0.0.0   -p 8822 -C "exit"
 
 # Copying configurations files to the Hadoop folder 
-cp $(pwd)/resources/configs/hadoop/${HADOOP_VERSION}/core-site.xml ${HADOOP_HOME}/etc/hadoop/ 
-cp $(pwd)/resources/configs/hadoop/${HADOOP_VERSION}/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/
+cp ~/resources/configs/hadoop/${HADOOP_VERSION}/core-site.xml ${HADOOP_HOME}/etc/hadoop/ 
+cp ~/resources/configs/hadoop/${HADOOP_VERSION}/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/
 
 # Formatting the filesystem
 hdfs namenode -format -force -nonInteractive >> /dev/null
 
-start-dfs.sh                                                                                     && \
+start-dfs.sh                                                                                && \
 #Copying YARN configurations for Pseudo-Distributed Mode
-cp $(pwd)/resources/configs/hadoop/${HADOOP_VERSION}/mapred-site.xml ${HADOOP_HOME}/etc/hadoop/  && \
-cp $(pwd)/resources/configs/hadoop/${HADOOP_VERSION}/yarn-site.xml   ${HADOOP_HOME}/etc/hadoop/  && \
+cp ~/resources/configs/hadoop/${HADOOP_VERSION}/mapred-site.xml ${HADOOP_HOME}/etc/hadoop/  && \
+cp ~/resources/configs/hadoop/${HADOOP_VERSION}/yarn-site.xml   ${HADOOP_HOME}/etc/hadoop/  && \
 
-start-yarn.sh                                                                                    && \
+start-yarn.sh                                                                               && \
 
-hdfs dfs -mkdir /user/                                                                           && \
-hdfs dfs -mkdir /user/matheus/                                                                   && \
-hdfs dfs -mkdir /user/$USER                                                                      &
+hdfs dfs -mkdir /user/                                                                      && \
+hdfs dfs -mkdir /user/matheus/                                                              && \
+hdfs dfs -mkdir /user/$USER                                                                 &
 
 # echo "${USER} $(pwd) ${HADOOP_HOME} ${HADOOP_VERSION}" > startresult.txt
diff --git a/binder/start.hbase b/binder/start.hbase
@@ -3,7 +3,7 @@
 # deleting slf4j lib. Hive will automatically use slf4j-log4j jar file present in Hadoop
 rm ${HBASE_HOME}/lib/client-facing-thirdparty/slf4j-log4j12-1.7.25.jar
 
-cp ./resources/configs/hbase/${HBASE_VERSION}/hbase-site.xml  ${HBASE_HOME}/conf/
+cp ~/resources/configs/hbase/${HBASE_VERSION}/hbase-site.xml  ${HBASE_HOME}/conf/
 
 hdfs dfs -mkdir /hbase
 

diff --git a/binder/start.hive b/binder/start.hive
@@ -1,8 +1,8 @@
 #!/bin/bash
 
 #setting log level higher for less verbosity
-cp ./resources/configs/hive/${HIVE_VERSION}/beeline-log4j2.properties ${HIVE_HOME}/conf/
-cp ./resources/configs/hive/${HIVE_VERSION}/hive-site.xml             ${HIVE_HOME}/conf/
+cp ~/resources/configs/hive/${HIVE_VERSION}/beeline-log4j2.properties ${HIVE_HOME}/conf/
+cp ~/resources/configs/hive/${HIVE_VERSION}/hive-site.xml             ${HIVE_HOME}/conf/
 
 #hack to make metastore_db within HIVE_HOME. 
 sed -i "s~HIVE_HOME~$HIVE_HOME~g" ${HIVE_HOME}/conf/hive-site.xml

diff --git a/binder/start.mysql b/binder/start.mysql
@@ -4,12 +4,17 @@
 
 $MYSQL_HOME/bin/mysqld  --user='${USER}' --initialize-insecure --skip-grant-tables
 
-cd     ${MYSQL_HOME}                 && \
-nohup  ${MYSQL_HOME}/bin/mysqld_safe &
+cd     ${MYSQL_HOME}          && \
+${MYSQL_HOME}/bin/mysqld_safe &
 
-sleep 5
+# waiting for mysqld startup
+while ! mysqladmin ping -h "localhost" --silent; do
+    echo "mysqld is down...";
+    sleep 5
+done
 
-$MYSQL_HOME/bin/mysql -u root -e "CREATE USER '${USER}'@'localhost';"
-$MYSQL_HOME/bin/mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '${USER}'@'localhost';"
+echo "mysqld is up!";
 
-touch done.txt
+# creating local user and granting all privileges
+$MYSQL_HOME/bin/mysql -u root -e "CREATE USER '${USER}'@'localhost';";
+$MYSQL_HOME/bin/mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '${USER}'@'localhost';" ;