diff --git a/binder/start b/binder/start index f8b370c..774895d 100644 --- a/binder/start +++ b/binder/start @@ -1,14 +1,22 @@ #!/bin/bash -chmod +x ./binder/start.hadoop -chmod +x ./binder/start.hive -chmod +x ./binder/start.mysql -chmod +x ./binder/start.hbase -#chmod +x ./binder/start.mariadb +chmod +x ~/binder/start.hadoop +chmod +x ~/binder/start.hive +chmod +x ~/binder/start.mysql +chmod +x ~/binder/start.hbase +#chmod +x ~/binder/start.mariadb +# starting mysql in parallel +~/binder/start.mysql & + +~/binder/start.hadoop && \ +sleep 3 && \ +~/binder/start.hive && \ +sleep 3 && \ +~/binder/start.hbase && \ +touch ready.txt & -./binder/start.hadoop && sleep 3 && ./binder/start.hive && sleep 3 && ./binder/start.hbase && ./binder/start.mysql & # DO NOT REMOVE(!) diff --git a/binder/start.hadoop b/binder/start.hadoop index fbbf26d..7e73d3b 100644 --- a/binder/start.hadoop +++ b/binder/start.hadoop @@ -2,33 +2,33 @@ #loading envvars #for auto load in binderhub: -source $(pwd)/.profile +source ~/.profile #for manual start: run #source $(pwd)/.bashrc #Starting sshd server -/usr/sbin/sshd -f $(pwd)/resources/configs/ssh/sshd_config +/usr/sbin/sshd -f ~/resources/configs/ssh/sshd_config # Adding names to know hosts. This step avoids yes/no host confirmation. ssh -o "StrictHostKeyChecking no" $USER@localhost -p 8822 -C "exit" ssh -o "StrictHostKeyChecking no" $USER@0.0.0.0 -p 8822 -C "exit" # Copying configurations files to the Hadoop folder -cp $(pwd)/resources/configs/hadoop/${HADOOP_VERSION}/core-site.xml ${HADOOP_HOME}/etc/hadoop/ -cp $(pwd)/resources/configs/hadoop/${HADOOP_VERSION}/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/ +cp ~/resources/configs/hadoop/${HADOOP_VERSION}/core-site.xml ${HADOOP_HOME}/etc/hadoop/ +cp ~/resources/configs/hadoop/${HADOOP_VERSION}/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/ # Formatting the filesystem hdfs namenode -format -force -nonInteractive >> /dev/null -start-dfs.sh && \ +start-dfs.sh && \ #Copying YARN configurations for Pseudo-Distributed Mode -cp $(pwd)/resources/configs/hadoop/${HADOOP_VERSION}/mapred-site.xml ${HADOOP_HOME}/etc/hadoop/ && \ -cp $(pwd)/resources/configs/hadoop/${HADOOP_VERSION}/yarn-site.xml ${HADOOP_HOME}/etc/hadoop/ && \ +cp ~/resources/configs/hadoop/${HADOOP_VERSION}/mapred-site.xml ${HADOOP_HOME}/etc/hadoop/ && \ +cp ~/resources/configs/hadoop/${HADOOP_VERSION}/yarn-site.xml ${HADOOP_HOME}/etc/hadoop/ && \ -start-yarn.sh && \ +start-yarn.sh && \ -hdfs dfs -mkdir /user/ && \ -hdfs dfs -mkdir /user/matheus/ && \ -hdfs dfs -mkdir /user/$USER & +hdfs dfs -mkdir /user/ && \ +hdfs dfs -mkdir /user/matheus/ && \ +hdfs dfs -mkdir /user/$USER & # echo "${USER} $(pwd) ${HADOOP_HOME} ${HADOOP_VERSION}" > startresult.txt \ No newline at end of file diff --git a/binder/start.hbase b/binder/start.hbase index 852ce96..fa72825 100644 --- a/binder/start.hbase +++ b/binder/start.hbase @@ -3,7 +3,7 @@ # deleting slf4j lib. Hive will automatically use slf4j-log4j jar file present in Hadoop rm ${HBASE_HOME}/lib/client-facing-thirdparty/slf4j-log4j12-1.7.25.jar -cp ./resources/configs/hbase/${HBASE_VERSION}/hbase-site.xml ${HBASE_HOME}/conf/ +cp ~/resources/configs/hbase/${HBASE_VERSION}/hbase-site.xml ${HBASE_HOME}/conf/ hdfs dfs -mkdir /hbase diff --git a/binder/start.hive b/binder/start.hive index ab9598e..09b2f9b 100644 --- a/binder/start.hive +++ b/binder/start.hive @@ -1,8 +1,8 @@ #!/bin/bash #setting log level higher for less verbosity -cp ./resources/configs/hive/${HIVE_VERSION}/beeline-log4j2.properties ${HIVE_HOME}/conf/ -cp ./resources/configs/hive/${HIVE_VERSION}/hive-site.xml ${HIVE_HOME}/conf/ +cp ~/resources/configs/hive/${HIVE_VERSION}/beeline-log4j2.properties ${HIVE_HOME}/conf/ +cp ~/resources/configs/hive/${HIVE_VERSION}/hive-site.xml ${HIVE_HOME}/conf/ #hack to make metastore_db within HIVE_HOME. sed -i "s~HIVE_HOME~$HIVE_HOME~g" ${HIVE_HOME}/conf/hive-site.xml diff --git a/binder/start.mysql b/binder/start.mysql index c9bd437..8b53fef 100644 --- a/binder/start.mysql +++ b/binder/start.mysql @@ -4,12 +4,17 @@ $MYSQL_HOME/bin/mysqld --user='${USER}' --initialize-insecure --skip-grant-tables -cd ${MYSQL_HOME} && \ -nohup ${MYSQL_HOME}/bin/mysqld_safe & +cd ${MYSQL_HOME} && \ +${MYSQL_HOME}/bin/mysqld_safe & -sleep 5 +# waiting for mysqld startup +while ! mysqladmin ping -h "localhost" --silent; do + echo "mysqld is down..."; + sleep 5 +done -$MYSQL_HOME/bin/mysql -u root -e "CREATE USER '${USER}'@'localhost';" -$MYSQL_HOME/bin/mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '${USER}'@'localhost';" +echo "mysqld is up!"; -touch done.txt \ No newline at end of file +# creating local user and granting all privileges +$MYSQL_HOME/bin/mysql -u root -e "CREATE USER '${USER}'@'localhost';"; +$MYSQL_HOME/bin/mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO '${USER}'@'localhost';" ;