diff --git a/README.md b/README.md index 40238a0e..b8717fb3 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ The OVA is shipped as a bootstrapped Install Node. It must be cloned multiple t ##### OVA Download Links -* [dellemc-ecsce-3.0.0.1-install-node-2.2.1-vm0.ova](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.1-install-node-2.2.1-vm0.ova) -* [dellemc-ecsce-3.0.0.1-install-node-2.2.1-vm0.ova.xz](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.1-install-node-2.2.1-vm0.ova.xz) +* [dellemc-ecsce-3.0.0.2-install-node-2.2.2-vm0.ova](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.2-install-node-2.2.2-vm0.ova) +* [dellemc-ecsce-3.0.0.2-install-node-2.2.2-vm0.ova.xz](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.2-install-node-2.2.2-vm0.ova.xz) #### [ECS Single-Node Deployment with Install Node (recommended)](docs/source/installation/ECS-Installation.md) Using an Install Node for isolated environments, deploy a stand-alone instance of ECS to a single hardware or virtual machine. diff --git a/bootstrap.sh b/bootstrap.sh index 86b7407a..be1fa158 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -310,6 +310,7 @@ o "" v "Creating shell shims in ${HOME}/bin for CLI commands" p Installing CLI shims symlink_scripts +update_path_in_bashrc ### Override nameservers provided by DHCP if -o was given. diff --git a/bootstrap_plugins/centos72.plugin.sh b/bootstrap_plugins/centos72.plugin.sh index 320b38bf..deb78a35 100644 --- a/bootstrap_plugins/centos72.plugin.sh +++ b/bootstrap_plugins/centos72.plugin.sh @@ -26,34 +26,31 @@ do_preflight() { } # packages to install before others -#list_prefix_packages='epel-release python-devel wget curl ntp' -list_prefix_packages='wget curl ntp epel-release yum-utils' +list_prefix_packages='wget curl epel-release yum-utils' # script to run for installing prefix_packages in_prefix_packages() { in_repo_pkg "$list_prefix_packages" - if ! docker version; then - curl -fsSL https://get.docker.com/ | sudo sh - fi - sudo systemctl enable docker - sudo systemctl start docker - sudo usermod -aG docker $(whoami) } # packages to install # list_general_packages='yum-utils git python-pip python-docker-py' -list_general_packages='git python-docker-py' +list_general_packages='git ntp docker' # script to run for installing general_packages in_general_packages() { in_repo_pkg "$list_general_packages" -# sudo pip install --upgrade pip -# sudo pip install --upgrade virtualenv +# if ! docker version; then +# curl -fsSL https://get.docker.com/ | sudo sh +# fi + sudo systemctl enable docker + sudo systemctl start docker + sudo usermod -aG docker $(whoami) } # packages to install after others -# list_suffix_packages='vim htop iotop iftop jq rsync pigz gdisk aria2' -list_suffix_packages='htop jq pigz gdisk aria2 firewalld' +list_suffix_packages='vim htop iotop iftop jq rsync pigz gdisk aria2 python-docker-py' +# list_suffix_packages='htop jq pigz gdisk aria2 python-docker-py' # script to run for installing suffix_packages in_suffix_packages() { diff --git a/bootstrap_plugins/centos73.plugin.sh b/bootstrap_plugins/centos73.plugin.sh index 172779a4..e03502cf 100755 --- a/bootstrap_plugins/centos73.plugin.sh +++ b/bootstrap_plugins/centos73.plugin.sh @@ -18,7 +18,6 @@ docker_binary='/bin/docker' # packages to clean up during preflight list_preflight_packages="git" -#nfs-tools" # Do any OS-specific tasks that must be done prior to bootstrap do_preflight() { @@ -26,34 +25,31 @@ do_preflight() { } # packages to install before others -#list_prefix_packages='epel-release python-devel wget curl ntp' -list_prefix_packages='wget curl ntp epel-release yum-utils' +list_prefix_packages='wget curl epel-release yum-utils' # script to run for installing prefix_packages in_prefix_packages() { in_repo_pkg "$list_prefix_packages" - if ! docker version; then - curl -fsSL https://get.docker.com/ | sudo sh - fi - sudo systemctl enable docker - sudo systemctl start docker - sudo usermod -aG docker $(whoami) } # packages to install # list_general_packages='yum-utils git python-pip python-docker-py' -list_general_packages='git python-docker-py' +list_general_packages='git ntp docker' # script to run for installing general_packages in_general_packages() { in_repo_pkg "$list_general_packages" -# sudo pip install --upgrade pip -# sudo pip install --upgrade virtualenv +# if ! docker version; then +# curl -fsSL https://get.docker.com/ | sudo sh +# fi + sudo systemctl enable docker + sudo systemctl start docker + sudo usermod -aG docker $(whoami) } # packages to install after others -# list_suffix_packages='vim htop iotop iftop jq rsync pigz gdisk aria2' -list_suffix_packages='htop jq pigz gdisk aria2' +list_suffix_packages='vim htop iotop iftop jq rsync pigz gdisk aria2 python-docker-py' +# list_suffix_packages='htop jq pigz gdisk aria2 python-docker-py' # script to run for installing suffix_packages in_suffix_packages() { @@ -76,12 +72,14 @@ in_vm_packages() { # command to install one or more os package manager package in_repo_pkg() { while ! sudo yum -y install $*; do - sleep 5 + sleep 1 done } rm_repo_pkg() { - sudo yum -y autoremove $* + while ! sudo yum -y autoremove $*; do + sleep 1 + done } # command to update all packages in the os package manager @@ -92,7 +90,7 @@ up_repo_pkg_all() { # command to rebuild the os package manager's database up_repo_db() { while ! sudo yum -y makecache; do - sleep 10 + sleep 1 # retry done } diff --git a/bootstrap_plugins/plugin-defaults.plugin.sh b/bootstrap_plugins/plugin-defaults.plugin.sh index fc454184..d5bf1068 100755 --- a/bootstrap_plugins/plugin-defaults.plugin.sh +++ b/bootstrap_plugins/plugin-defaults.plugin.sh @@ -23,3 +23,10 @@ symlink_scripts() { ln -s "${root}/ui/run.sh" "${HOME}/bin/$l" 2>/dev/null done } + +unset update_path_in_bashrc +update_path_in_bashrc() { + log "sed error is OK here if the proxy config file does not yet exist." + sudo sed -i -e '/PATH/d' $HOME/.bashrc + echo 'export PATH=$PATH:$HOME/.local/bin:$HOME/bin' >> $HOME/.bashrc +} diff --git a/patches/3.0.0.0-86889.0a0ee19/Dockerfile b/patches/3.0.0.0-86889.0a0ee19/Dockerfile new file mode 100644 index 00000000..6a2d0231 --- /dev/null +++ b/patches/3.0.0.0-86889.0a0ee19/Dockerfile @@ -0,0 +1,14 @@ +# Fixes to the default 3.0 HF2 image. +FROM emcvipr/object:3.0.0.0-86889.0a0ee19-reduced + +# Increase memory for transformsvc +ADD transformsvc /opt/storageos/bin/ + +# Fix disk partitioning script +ADD storageserver-partition-config.sh /opt/storageos/bin/ +RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh + +# Make vnest use separate thread pools to prevent deadlock +ADD vnest.object.properties /opt/storageos/conf/ + +RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f diff --git a/patches/3.0.0.0-86889.0a0ee19/storageserver-partition-config.sh b/patches/3.0.0.0-86889.0a0ee19/storageserver-partition-config.sh new file mode 100644 index 00000000..dbb9a0b4 --- /dev/null +++ b/patches/3.0.0.0-86889.0a0ee19/storageserver-partition-config.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# Copyright (c) 2013 EMC Corporation +# All Rights Reserved +# +# This software contains the intellectual property of EMC Corporation +# or is licensed to EMC Corporation from third parties. Use of this +# software and the intellectual property contained therein is expressly +# limited to the terms and conditions of the License Agreement under which +# it is provided by or on behalf of EMC. +# + +date + +# This script will generate the partition file (ss-partition-conf.json) to be used on this server if one is not found. +# Also a general configuration file will be generated with main parameters and a ss uuid. +# Assumes this is running on gen 3 hardware or a vipr data test node using disks mounted with a +# /dae/uuid-58f917ab-23ba-457f-a9df-d0f4b0597e2d type pattern or will look for /data (vipr) + +export configdir="/opt/storageos/conf" + +export p_configfile="$configdir/storageserver-partition-conf.json" +export configfile="$configdir/storageserver.conf" + +blockbinsizegb=10 +vm_bb_cout=5 +vm_part_count=5 + +function setopt() { + opt=$1 + val=$2 + if grep -q "$opt" "$configfile" ; then + sed -i "s#$opt=.*#$opt=$val#g" "$configfile" + else + echo "$opt=$val" >> $configfile + fi + +} + +echo "creating main config file: $configfile" +if [ ! -e "$configfile" ]; then + echo "creating $configfile" + mkdir -p "$configdir" + cat << EOF > "$configfile" +listenaddress=0.0.0.0 +port=9099 +iothreads=100 +blockbinsizegb=$blockbinsizegb +debugenabled=false +logtoconsole=false +logtosyslog=true +maxoutstandingrequests=0 +initblockbins=0 +netserverListenPort=9069 +netserverLogLevel=INFO +ioThreadNumber=200 +perfCountEnabled=true +maxMemThresholdKb=3145728 +sendQHighWaterMark=1000 +recvThreadNicePriority=-15 +sendThreadNicePriority=-15 +enableAffinity=true +partitionroot=/dae +agentUrlEndPoint=/host/data/agent.json +agentDisksRestFragment=/v1/agent/node/storage/disk/disks +trustStore=/host/data/security/truststore.pem +verifyCerts=true +dbusEnabled=false +EOF +else + echo "$configfile already exists, no action taken" +fi + +if grep -q -E 'VMware.*Virtual.*disk' /proc/scsi/sg/device_strs && [ ! -e /data/is_community_edition ] +then + # this a virtual datanode, creating a json file with predefined values + echo "virtual node detected - using json config file" + setopt partitionconfig $p_configfile + setopt partitionroot=/dae + root="/data/storageserver" + + for part in $(seq 1 $vm_part_count); do + for bb in $(seq 1 $vm_bb_cout); do + bb=$(printf "%0*d\n" 4 $bb) + mkdir -m 777 -p $root/uuid-$part + fallocate -l${blockbinsizegb}G $root/uuid-$part/$bb + chmod 777 $root/uuid-$part/$bb + done + done + + if [ ! -e "$p_configfile" ]; then + echo "creating $p_configfile" + mkdir -p "$configdir" + + printf '{\n%2s"disks": [\n' > $p_configfile + for part in $(seq 1 "$vm_part_count"); do + printf '%4s{\n%6s"uuid": "'$part'",\n%6s"health": "Good"\n%4s}' >> $p_configfile + [ "$part" -lt "$vm_part_count" ] && echo "," >> $p_configfile || echo "" >> $p_configfile + done + printf '%2s]\n}' >> $p_configfile + + else + echo "$p_configfile already exists, no action taken" + fi +else + # this is a commodity hardware node. do not generate a json file - ss will use dbus to obtain partitions + echo "commodity node detected - using dbus api" +fi + +if ! grep --quiet "uuid" "$configfile"; then + echo "generating server uuid" + uuidgen="$(which uuidgen)" + if [ ! -z "$uuidgen" ]; then + uuid="$(uuidgen)" + else + echo "warning: uuidgen utility was not found on your system. using timestamp as a uuid" + uuid="$(date +%s)" + fi + setopt uuid "$uuid" +fi diff --git a/patches/3.0.0.0-86889.0a0ee19/transformsvc b/patches/3.0.0.0-86889.0a0ee19/transformsvc new file mode 100644 index 00000000..144f5edc --- /dev/null +++ b/patches/3.0.0.0-86889.0a0ee19/transformsvc @@ -0,0 +1,27 @@ +#!/bin/sh +LIB_DIR="/opt/storageos/lib" +export JAVA_HOME="/usr/lib64/jvm/java-1.8.0-oracle/jre" +export PATH="${JAVA_HOME}/bin:/opt/storageos/bin:/bin:/usr/bin" + +export CLASSPATH="/opt/storageos/conf:${LIB_DIR}:${LIB_DIR}/storageos-transformsvc.jar:${LIB_DIR}/rsa-bsafe-for-centera.jar:${LIB_DIR}/slf4j-api-1.7.5.jar:${LIB_DIR}/jul-to-slf4j-1.7.5.jar:${LIB_DIR}/jcl-over-slf4j-1.7.5.jar:${LIB_DIR}/log4j-core-2.5.jar:${LIB_DIR}/log4j-api-2.5.jar:${LIB_DIR}/log4j-slf4j-impl-2.5.jar:${LIB_DIR}/log4j-jcl-2.5.jar:${LIB_DIR}/joda-time-2.2.jar:${LIB_DIR}/super-csv-2.1.0.jar:${LIB_DIR}/antlr-runtime-3.2.jar:${LIB_DIR}/uuid-3.2.jar:${LIB_DIR}/cassandra-clientutil-2.0.10.jar:${LIB_DIR}/cassandra-thrift-2.0.10.jar:${LIB_DIR}/cassandra-all-2.0.10.jar:${LIB_DIR}/commons-lang3-3.1.jar:${LIB_DIR}/compress-lzf-0.8.4.jar:${LIB_DIR}/high-scale-lib-1.1.2.jar:${LIB_DIR}/jackson-mapper-asl-1.9.2.jar:${LIB_DIR}/jackson-core-asl-1.9.2.jar:${LIB_DIR}/jamm-0.2.5.jar:${LIB_DIR}/jbcrypt-0.3m.jar:${LIB_DIR}/jline-1.0.jar:${LIB_DIR}/json-simple-1.1.jar:${LIB_DIR}/lz4-1.2.0.jar:${LIB_DIR}/netty-3.6.6.Final.jar:${LIB_DIR}/metrics-core-2.2.0.jar:${LIB_DIR}/servlet-api-2.5.jar:${LIB_DIR}/snakeyaml-1.11.jar:${LIB_DIR}/snaptree-0.1.jar:${LIB_DIR}/antlr-3.2.jar:${LIB_DIR}/snappy-java-1.0.5.jar:${LIB_DIR}/concurrentlinkedhashmap-lru-1.3.jar:${LIB_DIR}/commons-cli-1.1.jar:${LIB_DIR}/libthrift-0.9.1.jar:${LIB_DIR}/astyanax-queue-1.56.49.jar:${LIB_DIR}/astyanax-recipes-1.56.49.jar:${LIB_DIR}/astyanax-cassandra-1.56.49.jar:${LIB_DIR}/astyanax-core-1.56.49.jar:${LIB_DIR}/astyanax-entity-mapper-1.56.49.jar:${LIB_DIR}/astyanax-thrift-1.56.49.jar:${LIB_DIR}/commons-lang-2.6.jar:${LIB_DIR}/aopalliance-1.0.jar:${LIB_DIR}/spring-aop-4.2.0.RELEASE.jar:${LIB_DIR}/spring-aspects-4.2.0.RELEASE.jar:${LIB_DIR}/spring-beans-4.2.0.RELEASE.jar:${LIB_DIR}/spring-context-4.2.0.RELEASE.jar:${LIB_DIR}/spring-context-support-4.2.0.RELEASE.jar:${LIB_DIR}/spring-core-4.2.0.RELEASE.jar:${LIB_DIR}/spring-expression-4.2.0.RELEASE.jar:${LIB_DIR}/spring-instrument-4.2.0.RELEASE.jar:${LIB_DIR}/spring-oxm-4.2.0.RELEASE.jar:${LIB_DIR}/spring-test-4.2.0.RELEASE.jar:${LIB_DIR}/spring-tx-4.2.0.RELEASE.jar:${LIB_DIR}/spring-web-4.2.0.RELEASE.jar:${LIB_DIR}/spring-webmvc-4.2.0.RELEASE.jar:${LIB_DIR}/spring-security-aspects-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-config-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-core-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-crypto-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-ldap-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-openid-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-remoting-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-taglibs-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-web-4.0.2.RELEASE.jar:${LIB_DIR}/spring-ldap-core-2.0.3.RELEASE.jar:${LIB_DIR}/netty-3.2.8.Final.jar:${LIB_DIR}/zookeeper-3.4.6-11-vipr.jar:${LIB_DIR}/curator-client-2.6.0.jar:${LIB_DIR}/curator-framework-2.6.0.jar:${LIB_DIR}/curator-recipes-2.6.0.jar:${LIB_DIR}/curator-test-2.6.0.jar:${LIB_DIR}/curator-x-discovery-2.6.0.jar:${LIB_DIR}/curator-x-discovery-server-2.6.0.jar:${LIB_DIR}/commons-logging-adapters-1.1.jar:${LIB_DIR}/commons-logging-api-1.1.jar:${LIB_DIR}/commons-httpclient-3.1.jar:${LIB_DIR}/asm-3.1.jar:${LIB_DIR}/jackson-jaxrs-1.9.2.jar:${LIB_DIR}/jackson-xc-1.9.2.jar:${LIB_DIR}/jersey-client-1.12.jar:${LIB_DIR}/jersey-core-1.12.jar:${LIB_DIR}/jersey-json-1.12.jar:${LIB_DIR}/jersey-server-1.12.jar:${LIB_DIR}/jersey-servlet-1.12.jar:${LIB_DIR}/jettison-1.1.jar:${LIB_DIR}/jsr311-api-1.1.1.jar:${LIB_DIR}/jersey-apache-client-1.12.jar:${LIB_DIR}/jersey-apache-client4-1.12.jar:${LIB_DIR}/jersey-multipart-1.12.jar:${LIB_DIR}/jetty-server-9.3.8.v20160314.jar:${LIB_DIR}/jetty-util-9.3.8.v20160314.jar:${LIB_DIR}/jetty-servlet-9.3.8.v20160314.jar:${LIB_DIR}/jetty-servlets-9.3.8.v20160314.jar:${LIB_DIR}/jetty-http-9.3.8.v20160314.jar:${LIB_DIR}/jetty-security-9.3.8.v20160314.jar:${LIB_DIR}/jetty-io-9.3.8.v20160314.jar:${LIB_DIR}/jetty-continuation-9.3.8.v20160314.jar:${LIB_DIR}/jetty-deploy-9.3.8.v20160314.jar:${LIB_DIR}/jetty-webapp-9.3.8.v20160314.jar:${LIB_DIR}/jetty-xml-9.3.8.v20160314.jar:${LIB_DIR}/commons-jexl-1.1.jar:${LIB_DIR}/perf4j-0.9.16.jar:${LIB_DIR}/aspectjweaver-1.7.0.jar:${LIB_DIR}/aspectjrt-1.7.0.jar:${LIB_DIR}/cglib-nodep-2.2.2.jar:${LIB_DIR}/httpcore-4.3.2.jar:${LIB_DIR}/httpclient-4.3.3.jar:${LIB_DIR}/jna-3.5.1.jar:${LIB_DIR}/platform-3.5.1.jar:${LIB_DIR}/netty-all-4.0.19.Final.jar:${LIB_DIR}/casnaming-2.1.0.2-832.jar:${LIB_DIR}/aws-java-sdk-core-1.9.30.jar:${LIB_DIR}/aws-java-sdk-s3-1.9.30.jar:${LIB_DIR}/storageos-xml.jar:${LIB_DIR}/storageos-common.jar:${LIB_DIR}/storageos-client.jar:${LIB_DIR}/storageos-controller.jar:${LIB_DIR}/storageos-objcontrolsvc.jar:${LIB_DIR}/storageos-provisionsvc.jar:${LIB_DIR}/storageos-directoryactions.jar:${LIB_DIR}/storageos-ownership.jar:${LIB_DIR}/storageos-devicescanner.jar:${LIB_DIR}/storageos-gc.jar:${LIB_DIR}/storageos-ssmclient.jar:${LIB_DIR}/storageos-sscommunicator.jar:${LIB_DIR}/storageos-shared.jar:${LIB_DIR}/storageos-faultinjection.jar:${LIB_DIR}/storageos-centeramapiclient.jar:${LIB_DIR}/storageos-blobclient.jar:${LIB_DIR}/storageos-vnest.jar:${LIB_DIR}/storageos-vnestclient.jar:${LIB_DIR}/storageos-transformclient.jar:${LIB_DIR}/storageos-cashead.jar:${LIB_DIR}/storageos-stat.jar:${LIB_DIR}/platforminterfaces-4.3.479.jar:${LIB_DIR}/protobuf-java-2.5.0.jar:${LIB_DIR}/storageos-logging.jar:${LIB_DIR}/fabric-agent-api-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/fabric-agent-client-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/fabric-base-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/fabric-lifecycle-api-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/fabric-lifecycle-client-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/cryptojce-6.1.2-FIPS-140.jar:${LIB_DIR}/cryptojcommon-6.1.2-FIPS-140.jar:${LIB_DIR}/jcmFIPS-6.1.2-FIPS-140.jar:${LIB_DIR}/certj-6.1.1-FIPS-140.jar:${LIB_DIR}/sslj-6.1.2-FIPS-140.jar:${LIB_DIR}/storageos-apisvc.jar:${LIB_DIR}/storageos-syssvc.jar:${LIB_DIR}/storageos-ec.jar:${LIB_DIR}/storageos-dynamicconfig.jar:${LIB_DIR}/storageos-daresecurity.jar:${LIB_DIR}/storageos-nfsclient.jar:${LIB_DIR}/FPLibrary-3.3.62.jar:${LIB_DIR}/commons-io-2.4.jar:${LIB_DIR}/storageos-datasecurity.jar:${LIB_DIR}/storageos-ipc.jar:${LIB_DIR}/storageos-netty.jar:${LIB_DIR}/storageos-keystone.jar:${LIB_DIR}/storageos-directoryclient.jar:${LIB_DIR}/storageos-casnaming.jar:${LIB_DIR}/storageos-resourceclient.jar:${LIB_DIR}/storageos-keypool.jar:${LIB_DIR}/storageos-metering-client.jar:${LIB_DIR}/storageos-statclient.jar:${LIB_DIR}/storageos-policy.jar:${LIB_DIR}/storageos-filesvcCommon.jar:${LIB_DIR}/quasar-core-0.7.5.jar:${LIB_DIR}/asm-5.1.jar:${LIB_DIR}/kryo-2.24.0.jar:${LIB_DIR}/comsat-okhttp-0.7.0.jar:${LIB_DIR}/storageos-directorytable.jar:${LIB_DIR}/storageos-zkstore.jar:${LIB_DIR}/storageos-monitoring.jar:${LIB_DIR}/commons-collections-3.2.1.jar:${LIB_DIR}/storageos-communication.jar:${LIB_DIR}/commons-pool2-2.2.jar:${LIB_DIR}/nile-apisvc_models-1.0.0.0.466.502508c.jar:${LIB_DIR}/cryptix-3.2.0.jar:${LIB_DIR}/bcpkix-jdk15on-1.52.jar:${LIB_DIR}/bcprov-ext-jdk15on-1.52.jar:${LIB_DIR}/bcprov-jdk16-1.46.jar:${LIB_DIR}/javax.ws.rs-api-2.0.1.jar:${LIB_DIR}/storageos-fastmd5.jar:${LIB_DIR}/jaxen-1.1.1.jar:${LIB_DIR}/dom4j-1.6.1.jar:${LIB_DIR}/junit-4.11.jar:${LIB_DIR}/hamcrest-core-1.3.jar:${LIB_DIR}/hamcrest-library-1.3.jar:${LIB_DIR}/mapi-4.3.519.jar:${LIB_DIR}/storageos-cas_security.jar:${LIB_DIR}/storageos-cas_common.jar:${LIB_DIR}/storageos-blobclient_common.jar:${LIB_DIR}/commons-configuration-1.10.jar:${LIB_DIR}/storageos-btree.jar:${LIB_DIR}/storageos-coordinatorsvc.jar:${LIB_DIR}/storageos-cas-datastore.jar:${LIB_DIR}/storageos-statistics.jar:${LIB_DIR}/storageos-npacket.jar:${LIB_DIR}/storageos-blobaddress.jar:${LIB_DIR}/storageos-agent.jar:${LIB_DIR}/storageos-bytes.jar:${LIB_DIR}/storageos-utils.jar:${LIB_DIR}/storageos-config.jar:${LIB_DIR}/storageos-blobsvc.jar:${LIB_DIR}/concurrent-1.3.3.jar:${LIB_DIR}/sisu-guice-2.1.7.jar:${LIB_DIR}/jersey-spring-1.12.jar:${LIB_DIR}/storageos-ecs-client.jar:${LIB_DIR}/storageos-controllersvc.jar:${LIB_DIR}/storageos-security.jar:${LIB_DIR}/commons-compress-1.4.1.jar:${LIB_DIR}/xz-1.0.jar:${LIB_DIR}/jema-api-0.jar:${LIB_DIR}/xercesImpl-2.9.1.jar:${LIB_DIR}/eccpresso-all-2.8.jar:${LIB_DIR}/elmjava-3.3.0.jar:${LIB_DIR}/flexlm-11.11.1.jar:${LIB_DIR}/storageos-backup.jar:${LIB_DIR}/storageos-dbclient.jar:${LIB_DIR}/okhttp-2.6.0.jar:${LIB_DIR}/okhttp-urlconnection-2.6.0.jar:${LIB_DIR}/okhttp-apache-2.6.0.jar:${LIB_DIR}/okio-1.6.0.jar:${LIB_DIR}/storageos-paxos.jar:${LIB_DIR}/storageos-zkbtree.jar:${LIB_DIR}/storageos-errorhandling.jar:${LIB_DIR}/storageos-models.jar:${LIB_DIR}/storageos-serviceutils.jar:${LIB_DIR}/storageos-md5.jar:${LIB_DIR}/storageos-directoryserver.jar:${LIB_DIR}/storageos-georeplayer.jar:${LIB_DIR}/javax.inject-1.jar:${LIB_DIR}/log4j-1.2.16.jar:${LIB_DIR}/apache-log4j-extras-1.1.jar:${LIB_DIR}/gson-2.1.jar:${LIB_DIR}/jsch-0.1.51.jar:${LIB_DIR}/storageos-geomodels.jar:${LIB_DIR}/mail-1.4.3.jar:${LIB_DIR}/javassist-3.18.0-GA.jar:${LIB_DIR}/storageos-jmx.jar:${LIB_DIR}/activation-1.1.jar:${LIB_DIR}/guava-15.0.jar:${LIB_DIR}/javax.servlet-api-3.1.0.jar:${LIB_DIR}/jackson-core-2.3.5.jar:${LIB_DIR}/jackson-databind-2.3.5.jar:${LIB_DIR}/jackson-annotations-2.3.5.jar:${LIB_DIR}/commons-codec-1.8.jar:${LIB_DIR}/commons-logging-1.2.jar:${LIB_DIR}/tools.jar" + +_wait_for_dbsvc_init() { + start_time=$(date +%s) + while [ ! -e /var/run/storageos/dbsvc_initialized -a $(($(date +%s) - ${start_time})) -lt 0 ] ; do + sleep 10 + done +} + +_wait_for_dbsvc_init +[ "${0##*/}" = "dbsvc" ] && rm -f /var/run/storageos/dbsvc_initialized + +# Save PID +pid_file_name=${0##*/} +pid_file_name=${pid_file_name%%-coverage} +echo $$ >/var/run/storageos/${pid_file_name}.pid + + +exec -a $0 ${JAVA_HOME}/bin/java -ea -server -d64 -Xmx512m -Dproduct.home="${PRODUCT_HOME:-/opt/storageos}" \ + -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/storageos/logs/transformsvc-$$.hprof -XX:+PrintGCDateStamps -XX:+PrintGCDetails \ + -Dlog4j.configurationFile=file:/opt/storageos/conf/transformsvc-log4j2.xml -XX:NewSize=300m -XX:MaxNewSize=300m -Dsun.net.inetaddr.ttl=0 -XX:MaxPermSize=128m -XX:MaxTenuringThreshold=8 -XX:+UseConcMarkSweepGC -XX:+ExplicitGCInvokesConcurrent -XX:CMSInitiatingOccupancyFraction=60 -XX:+UseCMSInitiatingOccupancyOnly -XX:+PrintGCApplicationConcurrentTime -XX:+PrintGCApplicationStoppedTime -XX:+PrintTenuringDistribution -XX:+PrintGCDateStamps -Xloggc:/opt/storageos/logs/transformsvc-gc-$((($(ls /opt/storageos/logs/transformsvc-gc-*.log.0* -t 2>/dev/null | head -n1 | sed 's/.*-gc-\([0-9]\).*/\1/')+1)%10)).log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=50M \ + com.emc.storageos.data.transformation.Main "${@}" diff --git a/patches/3.0.0.0-86889.0a0ee19/vnest.object.properties b/patches/3.0.0.0-86889.0a0ee19/vnest.object.properties new file mode 100644 index 00000000..4bbb948d --- /dev/null +++ b/patches/3.0.0.0-86889.0a0ee19/vnest.object.properties @@ -0,0 +1,126 @@ +# vnest cluster group size +object.QuorumSize=5 +# number of zk servers in the ensemble +# keep same as QuorumSize; it is a separate config so it can be adjusted for standalone +object.ZkEnsembleSize=5 +# ownership lease time +object.zkReconfigWaitTimeSecs=300 + +object.RetryInterval=10000 +# This retry limit is for heartbeat only +object.RetryLimit=0 +# retry limit for bootstrap +object.BootstrapRetryLimit=1 + +object.TcpConnTimeout=1000 +object.MembershipBroadcastIntervalSecs=60 +object.UpdateCoordConfigIntervalSecs=60 +#number of thread to start for bcast +object.BcastLoadPerThread=3 +#number of bcast retry +object.NumBcastRetry=1 +#bcast threshold for serial or parallel +object.BcastThreshold=100 + +# comm requests to other servers +object.PaxosClientTimeoutSecs=10 +object.PaxosSuspectedTimeoutMillis=3000 + +# Total number of pre-allocated files +object.NumOfFiles=10 +# Threshold: if worker crosses this threshold then a new maintenance task is triggered to create new journal files +object.TaskTriggerThreshold=6 +# max journal size +object.FileSizeBytes=52428800 + +#Configuration module parameters +object.GroupMembershipFileName=groupmembership.config +object.CoordServerFileName=coord-server +object.CoordClientFileName=coord-client + +# configurable root, base of all directories +object.VNestRoot=/data/vnest +# configurable immediate child of VNestRoot, supports multiple vnests on single node (standalone) +object.VNestId=vnest-main + +# children of /VNestRoot/VNestId +object.ConfigurationPath=configuration +object.JournalPreallocationPath=journal +object.VNestJournalDir=replay +object.VNestIndexDir=index +object.VNestLockBoxDir=lockbox +object.VNestIndexRootRecordDir=root +object.VNestTempWorkspaceDir=workspace +object.RecycleDir=recycle +object.ShadowDir=shadow +object.VNestBootstrapDir=bootstrap + +#Initial refresh timeout in sec +object.InitialRefreshTimeoutSec=5 + +# HeartBeat +object.WaitForInitSecs=900 +object.WaitForBootstrapSecs=3600 +object.UnreachableSecs=3600 +object.UnreachableLeaderSecs=300 +object.TempNodeDownSecs=120 +object.HeartBeatFrequencySecs=5 +object.HeartBeatStatusCheckFrequencySecs=5 + +# Note: this should be less than object.WaitForInitSecs +object.MaxTimeSpentInitializingSecs=600 + +# GC +object.PreviousIndexesToKeep=3 +object.IndexGCIntervalSecs=300 +object.ChunkGCIntervalSecs=500 +# delete (true) vs. move to 'recycle' (false) +object.SkipRecycleDir=false + +# retention period of stale btree pages +object.BTree.DaysToRetain=2 +object.BTree.RecycleBinDir=recycleBin +object.BTree.StalePageRetenionDays=15 +object.VNestStaleJournalRetentionDays=5 +object.BTreeGCScriptTimeoutSecs=7200 + +# index BFW +object.IndexMaxSizePerWrite=524288 + +object.WSScanIntervalSec=600 + +object.WSDeleteThresholdSec=3600 + +# provided by container +object.NetworkConfigPath=/host/data/network.json +object.NodeInfoPath=/host/data/topology.json + +# comm retry in vNest paxos layer, for things like DT not init, etc +object.vNestPaxosRetryCount=3 + + +object.ObjectLockMaxWaitSeconds=10 + +# time before key reconfiguration task scan +object.ReconfigScanIntervalSecs=60 +# wait time after key in transition +object.ReconfigScanTimeoutIntervalSecs=140 +object.ReconfigKeyPartitions=7 +object.ReconfigThreads=8 +object.ReconfigListMaxKeys=512 + +object.KeyPurgeInitialDelayMinutes=10 +object.KeyPurgeIntervalMinutes=360 + +object.OperationsToggledInitialDelayMinutes=5 +object.OperationsToggledIntervalMinutes=60 +object.OperationsToggledLengthSeconds=10 +object.OperationsToggleEnabled=true +object.DiagnosticsEnabled=false + + +# DT initialized, but can't create journal chunk +object.WaitForReadOnlySecs=60 + +# Use separate thread pools to prevent deadlock in vnest init +object.UseSeparateThreadPools=true diff --git a/ui/etc/config.yml b/ui/etc/config.yml index 140b26ab..c1f1c4e8 100644 --- a/ui/etc/config.yml +++ b/ui/etc/config.yml @@ -13,7 +13,7 @@ --- ui: name: ECS Community Edition Install Node - version: 2.2.1 + version: 2.2.2 host_root_dir: /opt/emc/ecs-install state_file: /opt/state.yml deploy_file: /opt/deploy.yml diff --git a/ui/etc/release.conf b/ui/etc/release.conf index 04233a69..c3a69f1d 100644 --- a/ui/etc/release.conf +++ b/ui/etc/release.conf @@ -32,7 +32,7 @@ image_name='ecs-install' tag='latest' ver_maj='2' ver_min='2' -ver_rev='1' +ver_rev='2' ver_tag='r' serial=0 diff --git a/ui/libexec/console.sh b/ui/libexec/console.sh index 3ff10553..9eb144ce 100755 --- a/ui/libexec/console.sh +++ b/ui/libexec/console.sh @@ -9,6 +9,24 @@ # limited to the terms and conditions of the License Agreement under which # it is provided by or on behalf of EMC. +### TTY Detection + +export IS_TTY=false +export IS_PIPE=false +export IS_REDIRECTION=false + +if [[ -t 1 ]]; then + export IS_TTY=true +fi + +if [[ -p /dev/stdout ]]; then + export IS_PIPE=true +fi + +if [[ ! -t 1 && ! -p /dev/stdout ]]; then + export IS_REDIRECTION=true +fi + ### Logging and console output helpers quiet_flag=false diff --git a/ui/run.sh b/ui/run.sh index cf822982..0f899235 100755 --- a/ui/run.sh +++ b/ui/run.sh @@ -35,8 +35,17 @@ fi run() { run="${1}" shift - sudo docker run --rm -it --privileged --net=host ${default_mount_opts[@]} ${image_release} ${run} ${@} + + local _interactive='' + if ${IS_TTY}; then + _interactive='-t' + fi + + sudo docker run --rm -i ${_interactive} --privileged --net=host \ + ${default_mount_opts[@]} ${image_release} \ + ${run} ${@} rc=$? + o "" return ${rc} } @@ -100,6 +109,7 @@ case "$(basename ${0})" in run ecsdeploy access check || exit $? run ecsdeploy bootstrap || exit $? run ecsdeploy reboot || exit $? + sleep 10 run ping_until_clear run ecsdeploy deploy || exit $? run ecsdeploy start || exit $?