From 7031af20397025303eddc5596c0ab4742ac5f24c Mon Sep 17 00:00:00 2001 From: Travis Wichert Date: Wed, 28 Jun 2017 10:42:24 -0400 Subject: [PATCH 1/6] Resolves #287: Step 1 fails when there is no tty (#298) * Add shell shim path to PATH in .bashrc also * Add tty context detection * select '-it' in docker run/exec based on TTY presence * keep -i no matter what. --- bootstrap.sh | 1 + bootstrap_plugins/plugin-defaults.plugin.sh | 7 +++++++ ui/libexec/console.sh | 18 ++++++++++++++++++ ui/run.sh | 11 ++++++++++- 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index 86b7407a..be1fa158 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -310,6 +310,7 @@ o "" v "Creating shell shims in ${HOME}/bin for CLI commands" p Installing CLI shims symlink_scripts +update_path_in_bashrc ### Override nameservers provided by DHCP if -o was given. diff --git a/bootstrap_plugins/plugin-defaults.plugin.sh b/bootstrap_plugins/plugin-defaults.plugin.sh index fc454184..d5bf1068 100755 --- a/bootstrap_plugins/plugin-defaults.plugin.sh +++ b/bootstrap_plugins/plugin-defaults.plugin.sh @@ -23,3 +23,10 @@ symlink_scripts() { ln -s "${root}/ui/run.sh" "${HOME}/bin/$l" 2>/dev/null done } + +unset update_path_in_bashrc +update_path_in_bashrc() { + log "sed error is OK here if the proxy config file does not yet exist." + sudo sed -i -e '/PATH/d' $HOME/.bashrc + echo 'export PATH=$PATH:$HOME/.local/bin:$HOME/bin' >> $HOME/.bashrc +} diff --git a/ui/libexec/console.sh b/ui/libexec/console.sh index 3ff10553..9eb144ce 100755 --- a/ui/libexec/console.sh +++ b/ui/libexec/console.sh @@ -9,6 +9,24 @@ # limited to the terms and conditions of the License Agreement under which # it is provided by or on behalf of EMC. +### TTY Detection + +export IS_TTY=false +export IS_PIPE=false +export IS_REDIRECTION=false + +if [[ -t 1 ]]; then + export IS_TTY=true +fi + +if [[ -p /dev/stdout ]]; then + export IS_PIPE=true +fi + +if [[ ! -t 1 && ! -p /dev/stdout ]]; then + export IS_REDIRECTION=true +fi + ### Logging and console output helpers quiet_flag=false diff --git a/ui/run.sh b/ui/run.sh index cf822982..9f69c2ce 100755 --- a/ui/run.sh +++ b/ui/run.sh @@ -35,8 +35,17 @@ fi run() { run="${1}" shift - sudo docker run --rm -it --privileged --net=host ${default_mount_opts[@]} ${image_release} ${run} ${@} + + local _interactive='' + if ${IS_TTY}; then + _interactive='-t' + fi + + sudo docker run --rm -i ${_interactive} --privileged --net=host \ + ${default_mount_opts[@]} ${image_release} \ + ${run} ${@} rc=$? + o "" return ${rc} } From 5e817115d77c7639b4a4c64917445b3a9de385d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Moreno?= Date: Mon, 3 Jul 2017 20:35:20 +0200 Subject: [PATCH 2/6] Add patch for 3.0 HF2 (#300) This patch contains the build files for ECS 3.0.0.2 --- patches/3.0.0.0-86889.0a0ee19/Dockerfile | 14 ++ .../storageserver-partition-config.sh | 119 +++++++++++++++++ patches/3.0.0.0-86889.0a0ee19/transformsvc | 27 ++++ .../vnest.object.properties | 126 ++++++++++++++++++ 4 files changed, 286 insertions(+) create mode 100644 patches/3.0.0.0-86889.0a0ee19/Dockerfile create mode 100644 patches/3.0.0.0-86889.0a0ee19/storageserver-partition-config.sh create mode 100644 patches/3.0.0.0-86889.0a0ee19/transformsvc create mode 100644 patches/3.0.0.0-86889.0a0ee19/vnest.object.properties diff --git a/patches/3.0.0.0-86889.0a0ee19/Dockerfile b/patches/3.0.0.0-86889.0a0ee19/Dockerfile new file mode 100644 index 00000000..6a2d0231 --- /dev/null +++ b/patches/3.0.0.0-86889.0a0ee19/Dockerfile @@ -0,0 +1,14 @@ +# Fixes to the default 3.0 HF2 image. +FROM emcvipr/object:3.0.0.0-86889.0a0ee19-reduced + +# Increase memory for transformsvc +ADD transformsvc /opt/storageos/bin/ + +# Fix disk partitioning script +ADD storageserver-partition-config.sh /opt/storageos/bin/ +RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh + +# Make vnest use separate thread pools to prevent deadlock +ADD vnest.object.properties /opt/storageos/conf/ + +RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ true' $f diff --git a/patches/3.0.0.0-86889.0a0ee19/storageserver-partition-config.sh b/patches/3.0.0.0-86889.0a0ee19/storageserver-partition-config.sh new file mode 100644 index 00000000..dbb9a0b4 --- /dev/null +++ b/patches/3.0.0.0-86889.0a0ee19/storageserver-partition-config.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# Copyright (c) 2013 EMC Corporation +# All Rights Reserved +# +# This software contains the intellectual property of EMC Corporation +# or is licensed to EMC Corporation from third parties. Use of this +# software and the intellectual property contained therein is expressly +# limited to the terms and conditions of the License Agreement under which +# it is provided by or on behalf of EMC. +# + +date + +# This script will generate the partition file (ss-partition-conf.json) to be used on this server if one is not found. +# Also a general configuration file will be generated with main parameters and a ss uuid. +# Assumes this is running on gen 3 hardware or a vipr data test node using disks mounted with a +# /dae/uuid-58f917ab-23ba-457f-a9df-d0f4b0597e2d type pattern or will look for /data (vipr) + +export configdir="/opt/storageos/conf" + +export p_configfile="$configdir/storageserver-partition-conf.json" +export configfile="$configdir/storageserver.conf" + +blockbinsizegb=10 +vm_bb_cout=5 +vm_part_count=5 + +function setopt() { + opt=$1 + val=$2 + if grep -q "$opt" "$configfile" ; then + sed -i "s#$opt=.*#$opt=$val#g" "$configfile" + else + echo "$opt=$val" >> $configfile + fi + +} + +echo "creating main config file: $configfile" +if [ ! -e "$configfile" ]; then + echo "creating $configfile" + mkdir -p "$configdir" + cat << EOF > "$configfile" +listenaddress=0.0.0.0 +port=9099 +iothreads=100 +blockbinsizegb=$blockbinsizegb +debugenabled=false +logtoconsole=false +logtosyslog=true +maxoutstandingrequests=0 +initblockbins=0 +netserverListenPort=9069 +netserverLogLevel=INFO +ioThreadNumber=200 +perfCountEnabled=true +maxMemThresholdKb=3145728 +sendQHighWaterMark=1000 +recvThreadNicePriority=-15 +sendThreadNicePriority=-15 +enableAffinity=true +partitionroot=/dae +agentUrlEndPoint=/host/data/agent.json +agentDisksRestFragment=/v1/agent/node/storage/disk/disks +trustStore=/host/data/security/truststore.pem +verifyCerts=true +dbusEnabled=false +EOF +else + echo "$configfile already exists, no action taken" +fi + +if grep -q -E 'VMware.*Virtual.*disk' /proc/scsi/sg/device_strs && [ ! -e /data/is_community_edition ] +then + # this a virtual datanode, creating a json file with predefined values + echo "virtual node detected - using json config file" + setopt partitionconfig $p_configfile + setopt partitionroot=/dae + root="/data/storageserver" + + for part in $(seq 1 $vm_part_count); do + for bb in $(seq 1 $vm_bb_cout); do + bb=$(printf "%0*d\n" 4 $bb) + mkdir -m 777 -p $root/uuid-$part + fallocate -l${blockbinsizegb}G $root/uuid-$part/$bb + chmod 777 $root/uuid-$part/$bb + done + done + + if [ ! -e "$p_configfile" ]; then + echo "creating $p_configfile" + mkdir -p "$configdir" + + printf '{\n%2s"disks": [\n' > $p_configfile + for part in $(seq 1 "$vm_part_count"); do + printf '%4s{\n%6s"uuid": "'$part'",\n%6s"health": "Good"\n%4s}' >> $p_configfile + [ "$part" -lt "$vm_part_count" ] && echo "," >> $p_configfile || echo "" >> $p_configfile + done + printf '%2s]\n}' >> $p_configfile + + else + echo "$p_configfile already exists, no action taken" + fi +else + # this is a commodity hardware node. do not generate a json file - ss will use dbus to obtain partitions + echo "commodity node detected - using dbus api" +fi + +if ! grep --quiet "uuid" "$configfile"; then + echo "generating server uuid" + uuidgen="$(which uuidgen)" + if [ ! -z "$uuidgen" ]; then + uuid="$(uuidgen)" + else + echo "warning: uuidgen utility was not found on your system. using timestamp as a uuid" + uuid="$(date +%s)" + fi + setopt uuid "$uuid" +fi diff --git a/patches/3.0.0.0-86889.0a0ee19/transformsvc b/patches/3.0.0.0-86889.0a0ee19/transformsvc new file mode 100644 index 00000000..144f5edc --- /dev/null +++ b/patches/3.0.0.0-86889.0a0ee19/transformsvc @@ -0,0 +1,27 @@ +#!/bin/sh +LIB_DIR="/opt/storageos/lib" +export JAVA_HOME="/usr/lib64/jvm/java-1.8.0-oracle/jre" +export PATH="${JAVA_HOME}/bin:/opt/storageos/bin:/bin:/usr/bin" + +export CLASSPATH="/opt/storageos/conf:${LIB_DIR}:${LIB_DIR}/storageos-transformsvc.jar:${LIB_DIR}/rsa-bsafe-for-centera.jar:${LIB_DIR}/slf4j-api-1.7.5.jar:${LIB_DIR}/jul-to-slf4j-1.7.5.jar:${LIB_DIR}/jcl-over-slf4j-1.7.5.jar:${LIB_DIR}/log4j-core-2.5.jar:${LIB_DIR}/log4j-api-2.5.jar:${LIB_DIR}/log4j-slf4j-impl-2.5.jar:${LIB_DIR}/log4j-jcl-2.5.jar:${LIB_DIR}/joda-time-2.2.jar:${LIB_DIR}/super-csv-2.1.0.jar:${LIB_DIR}/antlr-runtime-3.2.jar:${LIB_DIR}/uuid-3.2.jar:${LIB_DIR}/cassandra-clientutil-2.0.10.jar:${LIB_DIR}/cassandra-thrift-2.0.10.jar:${LIB_DIR}/cassandra-all-2.0.10.jar:${LIB_DIR}/commons-lang3-3.1.jar:${LIB_DIR}/compress-lzf-0.8.4.jar:${LIB_DIR}/high-scale-lib-1.1.2.jar:${LIB_DIR}/jackson-mapper-asl-1.9.2.jar:${LIB_DIR}/jackson-core-asl-1.9.2.jar:${LIB_DIR}/jamm-0.2.5.jar:${LIB_DIR}/jbcrypt-0.3m.jar:${LIB_DIR}/jline-1.0.jar:${LIB_DIR}/json-simple-1.1.jar:${LIB_DIR}/lz4-1.2.0.jar:${LIB_DIR}/netty-3.6.6.Final.jar:${LIB_DIR}/metrics-core-2.2.0.jar:${LIB_DIR}/servlet-api-2.5.jar:${LIB_DIR}/snakeyaml-1.11.jar:${LIB_DIR}/snaptree-0.1.jar:${LIB_DIR}/antlr-3.2.jar:${LIB_DIR}/snappy-java-1.0.5.jar:${LIB_DIR}/concurrentlinkedhashmap-lru-1.3.jar:${LIB_DIR}/commons-cli-1.1.jar:${LIB_DIR}/libthrift-0.9.1.jar:${LIB_DIR}/astyanax-queue-1.56.49.jar:${LIB_DIR}/astyanax-recipes-1.56.49.jar:${LIB_DIR}/astyanax-cassandra-1.56.49.jar:${LIB_DIR}/astyanax-core-1.56.49.jar:${LIB_DIR}/astyanax-entity-mapper-1.56.49.jar:${LIB_DIR}/astyanax-thrift-1.56.49.jar:${LIB_DIR}/commons-lang-2.6.jar:${LIB_DIR}/aopalliance-1.0.jar:${LIB_DIR}/spring-aop-4.2.0.RELEASE.jar:${LIB_DIR}/spring-aspects-4.2.0.RELEASE.jar:${LIB_DIR}/spring-beans-4.2.0.RELEASE.jar:${LIB_DIR}/spring-context-4.2.0.RELEASE.jar:${LIB_DIR}/spring-context-support-4.2.0.RELEASE.jar:${LIB_DIR}/spring-core-4.2.0.RELEASE.jar:${LIB_DIR}/spring-expression-4.2.0.RELEASE.jar:${LIB_DIR}/spring-instrument-4.2.0.RELEASE.jar:${LIB_DIR}/spring-oxm-4.2.0.RELEASE.jar:${LIB_DIR}/spring-test-4.2.0.RELEASE.jar:${LIB_DIR}/spring-tx-4.2.0.RELEASE.jar:${LIB_DIR}/spring-web-4.2.0.RELEASE.jar:${LIB_DIR}/spring-webmvc-4.2.0.RELEASE.jar:${LIB_DIR}/spring-security-aspects-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-config-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-core-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-crypto-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-ldap-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-openid-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-remoting-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-taglibs-4.0.2.RELEASE.jar:${LIB_DIR}/spring-security-web-4.0.2.RELEASE.jar:${LIB_DIR}/spring-ldap-core-2.0.3.RELEASE.jar:${LIB_DIR}/netty-3.2.8.Final.jar:${LIB_DIR}/zookeeper-3.4.6-11-vipr.jar:${LIB_DIR}/curator-client-2.6.0.jar:${LIB_DIR}/curator-framework-2.6.0.jar:${LIB_DIR}/curator-recipes-2.6.0.jar:${LIB_DIR}/curator-test-2.6.0.jar:${LIB_DIR}/curator-x-discovery-2.6.0.jar:${LIB_DIR}/curator-x-discovery-server-2.6.0.jar:${LIB_DIR}/commons-logging-adapters-1.1.jar:${LIB_DIR}/commons-logging-api-1.1.jar:${LIB_DIR}/commons-httpclient-3.1.jar:${LIB_DIR}/asm-3.1.jar:${LIB_DIR}/jackson-jaxrs-1.9.2.jar:${LIB_DIR}/jackson-xc-1.9.2.jar:${LIB_DIR}/jersey-client-1.12.jar:${LIB_DIR}/jersey-core-1.12.jar:${LIB_DIR}/jersey-json-1.12.jar:${LIB_DIR}/jersey-server-1.12.jar:${LIB_DIR}/jersey-servlet-1.12.jar:${LIB_DIR}/jettison-1.1.jar:${LIB_DIR}/jsr311-api-1.1.1.jar:${LIB_DIR}/jersey-apache-client-1.12.jar:${LIB_DIR}/jersey-apache-client4-1.12.jar:${LIB_DIR}/jersey-multipart-1.12.jar:${LIB_DIR}/jetty-server-9.3.8.v20160314.jar:${LIB_DIR}/jetty-util-9.3.8.v20160314.jar:${LIB_DIR}/jetty-servlet-9.3.8.v20160314.jar:${LIB_DIR}/jetty-servlets-9.3.8.v20160314.jar:${LIB_DIR}/jetty-http-9.3.8.v20160314.jar:${LIB_DIR}/jetty-security-9.3.8.v20160314.jar:${LIB_DIR}/jetty-io-9.3.8.v20160314.jar:${LIB_DIR}/jetty-continuation-9.3.8.v20160314.jar:${LIB_DIR}/jetty-deploy-9.3.8.v20160314.jar:${LIB_DIR}/jetty-webapp-9.3.8.v20160314.jar:${LIB_DIR}/jetty-xml-9.3.8.v20160314.jar:${LIB_DIR}/commons-jexl-1.1.jar:${LIB_DIR}/perf4j-0.9.16.jar:${LIB_DIR}/aspectjweaver-1.7.0.jar:${LIB_DIR}/aspectjrt-1.7.0.jar:${LIB_DIR}/cglib-nodep-2.2.2.jar:${LIB_DIR}/httpcore-4.3.2.jar:${LIB_DIR}/httpclient-4.3.3.jar:${LIB_DIR}/jna-3.5.1.jar:${LIB_DIR}/platform-3.5.1.jar:${LIB_DIR}/netty-all-4.0.19.Final.jar:${LIB_DIR}/casnaming-2.1.0.2-832.jar:${LIB_DIR}/aws-java-sdk-core-1.9.30.jar:${LIB_DIR}/aws-java-sdk-s3-1.9.30.jar:${LIB_DIR}/storageos-xml.jar:${LIB_DIR}/storageos-common.jar:${LIB_DIR}/storageos-client.jar:${LIB_DIR}/storageos-controller.jar:${LIB_DIR}/storageos-objcontrolsvc.jar:${LIB_DIR}/storageos-provisionsvc.jar:${LIB_DIR}/storageos-directoryactions.jar:${LIB_DIR}/storageos-ownership.jar:${LIB_DIR}/storageos-devicescanner.jar:${LIB_DIR}/storageos-gc.jar:${LIB_DIR}/storageos-ssmclient.jar:${LIB_DIR}/storageos-sscommunicator.jar:${LIB_DIR}/storageos-shared.jar:${LIB_DIR}/storageos-faultinjection.jar:${LIB_DIR}/storageos-centeramapiclient.jar:${LIB_DIR}/storageos-blobclient.jar:${LIB_DIR}/storageos-vnest.jar:${LIB_DIR}/storageos-vnestclient.jar:${LIB_DIR}/storageos-transformclient.jar:${LIB_DIR}/storageos-cashead.jar:${LIB_DIR}/storageos-stat.jar:${LIB_DIR}/platforminterfaces-4.3.479.jar:${LIB_DIR}/protobuf-java-2.5.0.jar:${LIB_DIR}/storageos-logging.jar:${LIB_DIR}/fabric-agent-api-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/fabric-agent-client-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/fabric-base-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/fabric-lifecycle-api-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/fabric-lifecycle-client-1.3.0.0-2969.32ea651.jar:${LIB_DIR}/cryptojce-6.1.2-FIPS-140.jar:${LIB_DIR}/cryptojcommon-6.1.2-FIPS-140.jar:${LIB_DIR}/jcmFIPS-6.1.2-FIPS-140.jar:${LIB_DIR}/certj-6.1.1-FIPS-140.jar:${LIB_DIR}/sslj-6.1.2-FIPS-140.jar:${LIB_DIR}/storageos-apisvc.jar:${LIB_DIR}/storageos-syssvc.jar:${LIB_DIR}/storageos-ec.jar:${LIB_DIR}/storageos-dynamicconfig.jar:${LIB_DIR}/storageos-daresecurity.jar:${LIB_DIR}/storageos-nfsclient.jar:${LIB_DIR}/FPLibrary-3.3.62.jar:${LIB_DIR}/commons-io-2.4.jar:${LIB_DIR}/storageos-datasecurity.jar:${LIB_DIR}/storageos-ipc.jar:${LIB_DIR}/storageos-netty.jar:${LIB_DIR}/storageos-keystone.jar:${LIB_DIR}/storageos-directoryclient.jar:${LIB_DIR}/storageos-casnaming.jar:${LIB_DIR}/storageos-resourceclient.jar:${LIB_DIR}/storageos-keypool.jar:${LIB_DIR}/storageos-metering-client.jar:${LIB_DIR}/storageos-statclient.jar:${LIB_DIR}/storageos-policy.jar:${LIB_DIR}/storageos-filesvcCommon.jar:${LIB_DIR}/quasar-core-0.7.5.jar:${LIB_DIR}/asm-5.1.jar:${LIB_DIR}/kryo-2.24.0.jar:${LIB_DIR}/comsat-okhttp-0.7.0.jar:${LIB_DIR}/storageos-directorytable.jar:${LIB_DIR}/storageos-zkstore.jar:${LIB_DIR}/storageos-monitoring.jar:${LIB_DIR}/commons-collections-3.2.1.jar:${LIB_DIR}/storageos-communication.jar:${LIB_DIR}/commons-pool2-2.2.jar:${LIB_DIR}/nile-apisvc_models-1.0.0.0.466.502508c.jar:${LIB_DIR}/cryptix-3.2.0.jar:${LIB_DIR}/bcpkix-jdk15on-1.52.jar:${LIB_DIR}/bcprov-ext-jdk15on-1.52.jar:${LIB_DIR}/bcprov-jdk16-1.46.jar:${LIB_DIR}/javax.ws.rs-api-2.0.1.jar:${LIB_DIR}/storageos-fastmd5.jar:${LIB_DIR}/jaxen-1.1.1.jar:${LIB_DIR}/dom4j-1.6.1.jar:${LIB_DIR}/junit-4.11.jar:${LIB_DIR}/hamcrest-core-1.3.jar:${LIB_DIR}/hamcrest-library-1.3.jar:${LIB_DIR}/mapi-4.3.519.jar:${LIB_DIR}/storageos-cas_security.jar:${LIB_DIR}/storageos-cas_common.jar:${LIB_DIR}/storageos-blobclient_common.jar:${LIB_DIR}/commons-configuration-1.10.jar:${LIB_DIR}/storageos-btree.jar:${LIB_DIR}/storageos-coordinatorsvc.jar:${LIB_DIR}/storageos-cas-datastore.jar:${LIB_DIR}/storageos-statistics.jar:${LIB_DIR}/storageos-npacket.jar:${LIB_DIR}/storageos-blobaddress.jar:${LIB_DIR}/storageos-agent.jar:${LIB_DIR}/storageos-bytes.jar:${LIB_DIR}/storageos-utils.jar:${LIB_DIR}/storageos-config.jar:${LIB_DIR}/storageos-blobsvc.jar:${LIB_DIR}/concurrent-1.3.3.jar:${LIB_DIR}/sisu-guice-2.1.7.jar:${LIB_DIR}/jersey-spring-1.12.jar:${LIB_DIR}/storageos-ecs-client.jar:${LIB_DIR}/storageos-controllersvc.jar:${LIB_DIR}/storageos-security.jar:${LIB_DIR}/commons-compress-1.4.1.jar:${LIB_DIR}/xz-1.0.jar:${LIB_DIR}/jema-api-0.jar:${LIB_DIR}/xercesImpl-2.9.1.jar:${LIB_DIR}/eccpresso-all-2.8.jar:${LIB_DIR}/elmjava-3.3.0.jar:${LIB_DIR}/flexlm-11.11.1.jar:${LIB_DIR}/storageos-backup.jar:${LIB_DIR}/storageos-dbclient.jar:${LIB_DIR}/okhttp-2.6.0.jar:${LIB_DIR}/okhttp-urlconnection-2.6.0.jar:${LIB_DIR}/okhttp-apache-2.6.0.jar:${LIB_DIR}/okio-1.6.0.jar:${LIB_DIR}/storageos-paxos.jar:${LIB_DIR}/storageos-zkbtree.jar:${LIB_DIR}/storageos-errorhandling.jar:${LIB_DIR}/storageos-models.jar:${LIB_DIR}/storageos-serviceutils.jar:${LIB_DIR}/storageos-md5.jar:${LIB_DIR}/storageos-directoryserver.jar:${LIB_DIR}/storageos-georeplayer.jar:${LIB_DIR}/javax.inject-1.jar:${LIB_DIR}/log4j-1.2.16.jar:${LIB_DIR}/apache-log4j-extras-1.1.jar:${LIB_DIR}/gson-2.1.jar:${LIB_DIR}/jsch-0.1.51.jar:${LIB_DIR}/storageos-geomodels.jar:${LIB_DIR}/mail-1.4.3.jar:${LIB_DIR}/javassist-3.18.0-GA.jar:${LIB_DIR}/storageos-jmx.jar:${LIB_DIR}/activation-1.1.jar:${LIB_DIR}/guava-15.0.jar:${LIB_DIR}/javax.servlet-api-3.1.0.jar:${LIB_DIR}/jackson-core-2.3.5.jar:${LIB_DIR}/jackson-databind-2.3.5.jar:${LIB_DIR}/jackson-annotations-2.3.5.jar:${LIB_DIR}/commons-codec-1.8.jar:${LIB_DIR}/commons-logging-1.2.jar:${LIB_DIR}/tools.jar" + +_wait_for_dbsvc_init() { + start_time=$(date +%s) + while [ ! -e /var/run/storageos/dbsvc_initialized -a $(($(date +%s) - ${start_time})) -lt 0 ] ; do + sleep 10 + done +} + +_wait_for_dbsvc_init +[ "${0##*/}" = "dbsvc" ] && rm -f /var/run/storageos/dbsvc_initialized + +# Save PID +pid_file_name=${0##*/} +pid_file_name=${pid_file_name%%-coverage} +echo $$ >/var/run/storageos/${pid_file_name}.pid + + +exec -a $0 ${JAVA_HOME}/bin/java -ea -server -d64 -Xmx512m -Dproduct.home="${PRODUCT_HOME:-/opt/storageos}" \ + -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/storageos/logs/transformsvc-$$.hprof -XX:+PrintGCDateStamps -XX:+PrintGCDetails \ + -Dlog4j.configurationFile=file:/opt/storageos/conf/transformsvc-log4j2.xml -XX:NewSize=300m -XX:MaxNewSize=300m -Dsun.net.inetaddr.ttl=0 -XX:MaxPermSize=128m -XX:MaxTenuringThreshold=8 -XX:+UseConcMarkSweepGC -XX:+ExplicitGCInvokesConcurrent -XX:CMSInitiatingOccupancyFraction=60 -XX:+UseCMSInitiatingOccupancyOnly -XX:+PrintGCApplicationConcurrentTime -XX:+PrintGCApplicationStoppedTime -XX:+PrintTenuringDistribution -XX:+PrintGCDateStamps -Xloggc:/opt/storageos/logs/transformsvc-gc-$((($(ls /opt/storageos/logs/transformsvc-gc-*.log.0* -t 2>/dev/null | head -n1 | sed 's/.*-gc-\([0-9]\).*/\1/')+1)%10)).log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=50M \ + com.emc.storageos.data.transformation.Main "${@}" diff --git a/patches/3.0.0.0-86889.0a0ee19/vnest.object.properties b/patches/3.0.0.0-86889.0a0ee19/vnest.object.properties new file mode 100644 index 00000000..4bbb948d --- /dev/null +++ b/patches/3.0.0.0-86889.0a0ee19/vnest.object.properties @@ -0,0 +1,126 @@ +# vnest cluster group size +object.QuorumSize=5 +# number of zk servers in the ensemble +# keep same as QuorumSize; it is a separate config so it can be adjusted for standalone +object.ZkEnsembleSize=5 +# ownership lease time +object.zkReconfigWaitTimeSecs=300 + +object.RetryInterval=10000 +# This retry limit is for heartbeat only +object.RetryLimit=0 +# retry limit for bootstrap +object.BootstrapRetryLimit=1 + +object.TcpConnTimeout=1000 +object.MembershipBroadcastIntervalSecs=60 +object.UpdateCoordConfigIntervalSecs=60 +#number of thread to start for bcast +object.BcastLoadPerThread=3 +#number of bcast retry +object.NumBcastRetry=1 +#bcast threshold for serial or parallel +object.BcastThreshold=100 + +# comm requests to other servers +object.PaxosClientTimeoutSecs=10 +object.PaxosSuspectedTimeoutMillis=3000 + +# Total number of pre-allocated files +object.NumOfFiles=10 +# Threshold: if worker crosses this threshold then a new maintenance task is triggered to create new journal files +object.TaskTriggerThreshold=6 +# max journal size +object.FileSizeBytes=52428800 + +#Configuration module parameters +object.GroupMembershipFileName=groupmembership.config +object.CoordServerFileName=coord-server +object.CoordClientFileName=coord-client + +# configurable root, base of all directories +object.VNestRoot=/data/vnest +# configurable immediate child of VNestRoot, supports multiple vnests on single node (standalone) +object.VNestId=vnest-main + +# children of /VNestRoot/VNestId +object.ConfigurationPath=configuration +object.JournalPreallocationPath=journal +object.VNestJournalDir=replay +object.VNestIndexDir=index +object.VNestLockBoxDir=lockbox +object.VNestIndexRootRecordDir=root +object.VNestTempWorkspaceDir=workspace +object.RecycleDir=recycle +object.ShadowDir=shadow +object.VNestBootstrapDir=bootstrap + +#Initial refresh timeout in sec +object.InitialRefreshTimeoutSec=5 + +# HeartBeat +object.WaitForInitSecs=900 +object.WaitForBootstrapSecs=3600 +object.UnreachableSecs=3600 +object.UnreachableLeaderSecs=300 +object.TempNodeDownSecs=120 +object.HeartBeatFrequencySecs=5 +object.HeartBeatStatusCheckFrequencySecs=5 + +# Note: this should be less than object.WaitForInitSecs +object.MaxTimeSpentInitializingSecs=600 + +# GC +object.PreviousIndexesToKeep=3 +object.IndexGCIntervalSecs=300 +object.ChunkGCIntervalSecs=500 +# delete (true) vs. move to 'recycle' (false) +object.SkipRecycleDir=false + +# retention period of stale btree pages +object.BTree.DaysToRetain=2 +object.BTree.RecycleBinDir=recycleBin +object.BTree.StalePageRetenionDays=15 +object.VNestStaleJournalRetentionDays=5 +object.BTreeGCScriptTimeoutSecs=7200 + +# index BFW +object.IndexMaxSizePerWrite=524288 + +object.WSScanIntervalSec=600 + +object.WSDeleteThresholdSec=3600 + +# provided by container +object.NetworkConfigPath=/host/data/network.json +object.NodeInfoPath=/host/data/topology.json + +# comm retry in vNest paxos layer, for things like DT not init, etc +object.vNestPaxosRetryCount=3 + + +object.ObjectLockMaxWaitSeconds=10 + +# time before key reconfiguration task scan +object.ReconfigScanIntervalSecs=60 +# wait time after key in transition +object.ReconfigScanTimeoutIntervalSecs=140 +object.ReconfigKeyPartitions=7 +object.ReconfigThreads=8 +object.ReconfigListMaxKeys=512 + +object.KeyPurgeInitialDelayMinutes=10 +object.KeyPurgeIntervalMinutes=360 + +object.OperationsToggledInitialDelayMinutes=5 +object.OperationsToggledIntervalMinutes=60 +object.OperationsToggledLengthSeconds=10 +object.OperationsToggleEnabled=true +object.DiagnosticsEnabled=false + + +# DT initialized, but can't create journal chunk +object.WaitForReadOnlySecs=60 + +# Use separate thread pools to prevent deadlock in vnest init +object.UseSeparateThreadPools=true From 76b11c4cf574ccf1425a0b2972fac9a1506d1283 Mon Sep 17 00:00:00 2001 From: Travis Wichert Date: Mon, 3 Jul 2017 14:44:45 -0400 Subject: [PATCH 3/6] ECS-CommunityEdition-301 Update to ECS 3.0.0.2 (3.0.0 HF 2) (#302) --- README.md | 4 ++-- ui/etc/config.yml | 2 +- ui/etc/release.conf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 40238a0e..ddad8a6a 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ The OVA is shipped as a bootstrapped Install Node. It must be cloned multiple t ##### OVA Download Links -* [dellemc-ecsce-3.0.0.1-install-node-2.2.1-vm0.ova](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.1-install-node-2.2.1-vm0.ova) -* [dellemc-ecsce-3.0.0.1-install-node-2.2.1-vm0.ova.xz](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.1-install-node-2.2.1-vm0.ova.xz) +* [dellemc-ecsce-3.0.0.2-install-node-2.2.1-vm0.ova](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.2-install-node-2.2.1-vm0.ova) +* [dellemc-ecsce-3.0.0.2-install-node-2.2.1-vm0.ova.xz](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.2-install-node-2.2.1-vm0.ova.xz) #### [ECS Single-Node Deployment with Install Node (recommended)](docs/source/installation/ECS-Installation.md) Using an Install Node for isolated environments, deploy a stand-alone instance of ECS to a single hardware or virtual machine. diff --git a/ui/etc/config.yml b/ui/etc/config.yml index 140b26ab..7017cd4d 100644 --- a/ui/etc/config.yml +++ b/ui/etc/config.yml @@ -31,7 +31,7 @@ ui: ffx_sem: /opt/ffx.sem product: name: ECS - version: 3.0.0.1 + version: 3.0.0.2 vendor: Dell EMC flavor: Community Edition slogan: Free and Frictionless diff --git a/ui/etc/release.conf b/ui/etc/release.conf index 04233a69..97f888a6 100644 --- a/ui/etc/release.conf +++ b/ui/etc/release.conf @@ -8,7 +8,7 @@ # it is provided by or on behalf of EMC. release_name="ECS Community Edition" -release_version="3.0.0.1" +release_version="3.0.0.2" release_product="ECS Software" release_artifact="emccorp/ecs-software-3.0.0" release_tag="latest" From 558e19551ec91447b1b4cdfeee4c927fa095f4f5 Mon Sep 17 00:00:00 2001 From: Travis Wichert Date: Wed, 12 Jul 2017 18:54:09 -0400 Subject: [PATCH 4/6] revert to ECS CE 3.0.0.1 (3.0.0 HF1) (#308) --- ui/etc/config.yml | 2 +- ui/etc/release.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/etc/config.yml b/ui/etc/config.yml index 7017cd4d..140b26ab 100644 --- a/ui/etc/config.yml +++ b/ui/etc/config.yml @@ -31,7 +31,7 @@ ui: ffx_sem: /opt/ffx.sem product: name: ECS - version: 3.0.0.2 + version: 3.0.0.1 vendor: Dell EMC flavor: Community Edition slogan: Free and Frictionless diff --git a/ui/etc/release.conf b/ui/etc/release.conf index 97f888a6..04233a69 100644 --- a/ui/etc/release.conf +++ b/ui/etc/release.conf @@ -8,7 +8,7 @@ # it is provided by or on behalf of EMC. release_name="ECS Community Edition" -release_version="3.0.0.2" +release_version="3.0.0.1" release_product="ECS Software" release_artifact="emccorp/ecs-software-3.0.0" release_tag="latest" From da1e19cddea4e5ca1246a7f924d4faf9ad9b250d Mon Sep 17 00:00:00 2001 From: Travis Wichert Date: Wed, 12 Jul 2017 18:54:25 -0400 Subject: [PATCH 5/6] bugfix-docker-hang (#309) * Tweak the delay here for slow nodes. (cherry picked from commit c956a2debf10bb2f382d49e7ef092bf0c542994e) * Switch to mainline CentOS Docker rather than Docker's repo. Closes #307 --- bootstrap_plugins/centos72.plugin.sh | 23 +++++++++----------- bootstrap_plugins/centos73.plugin.sh | 32 +++++++++++++--------------- ui/run.sh | 1 + 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/bootstrap_plugins/centos72.plugin.sh b/bootstrap_plugins/centos72.plugin.sh index 320b38bf..deb78a35 100644 --- a/bootstrap_plugins/centos72.plugin.sh +++ b/bootstrap_plugins/centos72.plugin.sh @@ -26,34 +26,31 @@ do_preflight() { } # packages to install before others -#list_prefix_packages='epel-release python-devel wget curl ntp' -list_prefix_packages='wget curl ntp epel-release yum-utils' +list_prefix_packages='wget curl epel-release yum-utils' # script to run for installing prefix_packages in_prefix_packages() { in_repo_pkg "$list_prefix_packages" - if ! docker version; then - curl -fsSL https://get.docker.com/ | sudo sh - fi - sudo systemctl enable docker - sudo systemctl start docker - sudo usermod -aG docker $(whoami) } # packages to install # list_general_packages='yum-utils git python-pip python-docker-py' -list_general_packages='git python-docker-py' +list_general_packages='git ntp docker' # script to run for installing general_packages in_general_packages() { in_repo_pkg "$list_general_packages" -# sudo pip install --upgrade pip -# sudo pip install --upgrade virtualenv +# if ! docker version; then +# curl -fsSL https://get.docker.com/ | sudo sh +# fi + sudo systemctl enable docker + sudo systemctl start docker + sudo usermod -aG docker $(whoami) } # packages to install after others -# list_suffix_packages='vim htop iotop iftop jq rsync pigz gdisk aria2' -list_suffix_packages='htop jq pigz gdisk aria2 firewalld' +list_suffix_packages='vim htop iotop iftop jq rsync pigz gdisk aria2 python-docker-py' +# list_suffix_packages='htop jq pigz gdisk aria2 python-docker-py' # script to run for installing suffix_packages in_suffix_packages() { diff --git a/bootstrap_plugins/centos73.plugin.sh b/bootstrap_plugins/centos73.plugin.sh index 172779a4..e03502cf 100755 --- a/bootstrap_plugins/centos73.plugin.sh +++ b/bootstrap_plugins/centos73.plugin.sh @@ -18,7 +18,6 @@ docker_binary='/bin/docker' # packages to clean up during preflight list_preflight_packages="git" -#nfs-tools" # Do any OS-specific tasks that must be done prior to bootstrap do_preflight() { @@ -26,34 +25,31 @@ do_preflight() { } # packages to install before others -#list_prefix_packages='epel-release python-devel wget curl ntp' -list_prefix_packages='wget curl ntp epel-release yum-utils' +list_prefix_packages='wget curl epel-release yum-utils' # script to run for installing prefix_packages in_prefix_packages() { in_repo_pkg "$list_prefix_packages" - if ! docker version; then - curl -fsSL https://get.docker.com/ | sudo sh - fi - sudo systemctl enable docker - sudo systemctl start docker - sudo usermod -aG docker $(whoami) } # packages to install # list_general_packages='yum-utils git python-pip python-docker-py' -list_general_packages='git python-docker-py' +list_general_packages='git ntp docker' # script to run for installing general_packages in_general_packages() { in_repo_pkg "$list_general_packages" -# sudo pip install --upgrade pip -# sudo pip install --upgrade virtualenv +# if ! docker version; then +# curl -fsSL https://get.docker.com/ | sudo sh +# fi + sudo systemctl enable docker + sudo systemctl start docker + sudo usermod -aG docker $(whoami) } # packages to install after others -# list_suffix_packages='vim htop iotop iftop jq rsync pigz gdisk aria2' -list_suffix_packages='htop jq pigz gdisk aria2' +list_suffix_packages='vim htop iotop iftop jq rsync pigz gdisk aria2 python-docker-py' +# list_suffix_packages='htop jq pigz gdisk aria2 python-docker-py' # script to run for installing suffix_packages in_suffix_packages() { @@ -76,12 +72,14 @@ in_vm_packages() { # command to install one or more os package manager package in_repo_pkg() { while ! sudo yum -y install $*; do - sleep 5 + sleep 1 done } rm_repo_pkg() { - sudo yum -y autoremove $* + while ! sudo yum -y autoremove $*; do + sleep 1 + done } # command to update all packages in the os package manager @@ -92,7 +90,7 @@ up_repo_pkg_all() { # command to rebuild the os package manager's database up_repo_db() { while ! sudo yum -y makecache; do - sleep 10 + sleep 1 # retry done } diff --git a/ui/run.sh b/ui/run.sh index 9f69c2ce..0f899235 100755 --- a/ui/run.sh +++ b/ui/run.sh @@ -109,6 +109,7 @@ case "$(basename ${0})" in run ecsdeploy access check || exit $? run ecsdeploy bootstrap || exit $? run ecsdeploy reboot || exit $? + sleep 10 run ping_until_clear run ecsdeploy deploy || exit $? run ecsdeploy start || exit $? From ee137b3a758a3c23a0b5e4861b25488e7855c292 Mon Sep 17 00:00:00 2001 From: Travis Wichert Date: Wed, 12 Jul 2017 19:24:42 -0400 Subject: [PATCH 6/6] Release prep for 2.2.2 (#310) --- README.md | 4 ++-- ui/etc/config.yml | 2 +- ui/etc/release.conf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ddad8a6a..b8717fb3 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ The OVA is shipped as a bootstrapped Install Node. It must be cloned multiple t ##### OVA Download Links -* [dellemc-ecsce-3.0.0.2-install-node-2.2.1-vm0.ova](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.2-install-node-2.2.1-vm0.ova) -* [dellemc-ecsce-3.0.0.2-install-node-2.2.1-vm0.ova.xz](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.2-install-node-2.2.1-vm0.ova.xz) +* [dellemc-ecsce-3.0.0.2-install-node-2.2.2-vm0.ova](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.2-install-node-2.2.2-vm0.ova) +* [dellemc-ecsce-3.0.0.2-install-node-2.2.2-vm0.ova.xz](http://130852476153187606.public.ecstestdrive.com/public/dellemc-ecsce-3.0.0.2-install-node-2.2.2-vm0.ova.xz) #### [ECS Single-Node Deployment with Install Node (recommended)](docs/source/installation/ECS-Installation.md) Using an Install Node for isolated environments, deploy a stand-alone instance of ECS to a single hardware or virtual machine. diff --git a/ui/etc/config.yml b/ui/etc/config.yml index 140b26ab..c1f1c4e8 100644 --- a/ui/etc/config.yml +++ b/ui/etc/config.yml @@ -13,7 +13,7 @@ --- ui: name: ECS Community Edition Install Node - version: 2.2.1 + version: 2.2.2 host_root_dir: /opt/emc/ecs-install state_file: /opt/state.yml deploy_file: /opt/deploy.yml diff --git a/ui/etc/release.conf b/ui/etc/release.conf index 04233a69..c3a69f1d 100644 --- a/ui/etc/release.conf +++ b/ui/etc/release.conf @@ -32,7 +32,7 @@ image_name='ecs-install' tag='latest' ver_maj='2' ver_min='2' -ver_rev='1' +ver_rev='2' ver_tag='r' serial=0