update ECS image to latest 3.3 (3.3.0.2) (#487)

update OS compatibility to include CentOS 7.7

bootstrap.sh

@@ -529,19 +529,13 @@ o ""
 v "Installing bootstrap packages pass (1/3)"
 p Installing new packages
 in_prefix_packages 2>&1 | log
-#if $proxy_flag; then
-#    set_repo_proxy_idempotent
-#fi
 ping_sudo
 
 
 p Installing packages
 v "Installing bootstrap packages pass (2/3)"
 p Installing new packages
 in_general_packages 2>&1 | log
-#if $proxy_flag; then
-#    set_repo_proxy_idempotent
-#fi
 ping_sudo
 
 

bootstrap_plugins/centos77.plugin.sh

@@ -0,0 +1,226 @@
+#@IgnoreInspection BashAddShebang
+
+# Copyright (c) 2015 EMC Corporation
+# All Rights Reserved
+#
+# This software contains the intellectual property of EMC Corporation
+# or is licensed to EMC Corporation from third parties.  Use of this
+# software and the intellectual property contained therein is expressly
+# limited to the terms and conditions of the License Agreement under which
+# it is provided by or on behalf of EMC.
+
+# OS Support library for CentOS 7.5
+
+os_supported=true
+
+# Docker binary
+docker_binary='/bin/docker'
+
+# packages to clean up during preflight
+# Don't `yum autoremove curl`.  Yum is a dependency and it will throw errors.
+
+list_preflight_packages="git nfs-client nfs-tools rsync wget ntp docker vim pigz gdisk aria2 htop iotop iftop multitail dstat jq python-docker-py dkms qemu-guest-agent open-vm-tools open-vm-tools-desktop docker"
+
+# Do any OS-specific tasks that must be done prior to bootstrap
+do_preflight() {
+    rm_repo_pkg "$list_preflight_packages"
+}
+
+# packages to install before others
+list_prefix_packages='wget curl epel-release yum-utils device-mapper-persistent-data lvm2'
+
+# script to run for installing prefix_packages
+in_prefix_packages() {
+    in_repo_pkg "$list_prefix_packages"
+}
+
+# non-standard repos required for package installation
+additional_repos='https://download.docker.com/linux/centos/docker-ce.repo'
+
+
+# packages to install
+list_general_packages='git ntp vim rsync pigz gdisk aria2 yum-versionlock'
+list_docker_ce_packages='docker-ce-17.03.0.ce-1.el7.centos docker-ce-selinux-17.03.0.ce-1.el7.centos'
+
+# script to run for installing general_packages
+in_general_packages() {
+    add_repo "$additional_repos"
+    in_docker_ce "$list_docker_ce_packages"
+    in_repo_pkg "$list_general_packages"
+    sudo systemctl enable docker
+    sudo systemctl start docker
+    sudo usermod -aG docker $(whoami)
+}
+
+# packages to install after others
+list_suffix_packages='htop iotop iftop multitail dstat jq python-docker-py'
+# list_suffix_packages='htop jq pigz gdisk aria2 python-docker-py'
+
+# packages to lock after installation
+list_lock_packages='docker-ce-17.03.0.ce-1.el7.centos'
+
+# script to run for installing suffix_packages
+in_suffix_packages() {
+    in_repo_pkg "$list_suffix_packages"
+
+    # Install Rocker
+    curl -fsSL ${rocker_artifact_url} \
+    | sudo tar -xzC /usr/local/bin && sudo chmod +x /usr/local/bin/rocker
+}
+
+# packages to install if a VM
+list_vm_packages='dkms qemu-guest-agent open-vm-tools open-vm-tools-desktop'
+
+# command to run for installing vm_packages
+in_vm_packages() {
+    in_repo_pkg "$list_vm_packages"
+    # return 0
+}
+
+versionlock_packages() {
+    lock_pkg "$list_lock_packages"
+}
+
+# command to install one or more os package manager package
+in_repo_pkg() {
+    retry_with_timeout 10 300 sudo yum -y install $*
+}
+
+rm_repo_pkg() {
+    retry_with_timeout 10 300 sudo yum -y autoremove $*
+}
+
+# lock packages that we don't want updated
+lock_pkg() {
+    sudo yum versionlock $*
+}
+
+# add additional repositories to yum
+add_repo() {
+  sudo yum-config-manager --add-repo $*
+}
+
+# docker-ce packages required for older docker version
+in_docker_ce() {
+    sudo yum install -y --setopt=obsoletes=0 $*
+}
+
+# command to update all packages in the os package manager
+up_repo_pkg_all() {
+    retry_with_timeout 10 300 sudo yum -y update
+}
+
+# command to rebuild the os package manager's database
+up_repo_db() {
+    retry_with_timeout 10 300 sudo yum -y makecache
+}
+
+# command to set os package manager proxy
+set_repo_proxy_conf() {
+    sudo sed -i -e '/^proxy=/d' /etc/yum.conf
+    echo "proxy=${http_proxy}" \
+        | append /etc/yum.conf
+}
+
+# command to set os package manager to keep its cache
+set_repo_keepcache_conf() {
+    sudo sed -i -e '/^keepcache=/d' /etc/yum.conf
+    echo "keepcache=1" \
+        | append /etc/yum.conf
+}
+
+# idempotent config script to fixup repos to properly use proxycaches
+set_repo_cacheable_idempotent() {
+    sudo sed -i -e 's/^#baseurl=/baseurl=/' /etc/yum.repos.d/*
+    sudo sed -i -e 's/^mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/*
+}
+
+set_repo_mirror_idempotent() {
+    # sudo sed -i -e "s#http:///centos#http://${mirror_val}/centos#g" /etc/yum.repos.d/*
+    sudo sed -i -e "s#http://.*/centos#http://${mirror_val}/centos#g" /etc/yum.repos.d/*
+}
+
+# command to set the proxy for the whole OS
+set_os_proxy() {
+    sudo sed -i -e '/_proxy/d' /etc/environment
+    echo -n "http_proxy=${http_proxy}\nhttps_proxy=${http_proxy}\nftp_proxy=${http_proxy}\n" \
+        | append /etc/environment
+    if $mirror_flag; then
+        echo -n "no_proxy=${mirror_val}\n" | append /etc/environment
+    fi
+}
+
+# command to determine if the OS needs restarting after package updates
+get_os_needs_restarting() {
+    if ! [ -z "$(sudo /usr/bin/needs-restarting)" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# command to reboot the system
+do_reboot() {
+    sudo reboot
+}
+
+# Command to configure docker's proxy under centos flavored systemd
+set_docker_proxy() {
+    local tmpconf="/etc/systemd/system/docker.service.d/http-proxy.conf"
+    if ! [ -d "$(dirname $tmpconf)" ]; then
+        sudo mkdir "$(dirname $tmpconf)"
+    fi
+    log "sed error is OK here if the proxy config file does not yet exist."
+    sudo sed -i -e '/HTTP_PROXY/d' "$tmpconf"
+    echo "Environment=\"HTTP_PROXY=${http_proxy}\" \"NO_PROXY=localhost,127.0.0.1,$(hostname),$(hostname -f)\"" \
+        | append "$tmpconf"
+    sudo systemctl daemon-reload
+    sudo systemctl restart docker
+    sudo systemctl status docker
+}
+
+# command to add mitm cert to docker trust store
+set_docker_reg_cert() {
+    local registry="${1}"
+    local cert="${2}"
+    if ! [ -d "/etc/docker/certs.d/${registry}" ]; then
+        sudo mkdir -p "/etc/docker/certs.d/${registry}"
+        sudo cp "${cert}" "/etc/docker/certs.d/${registry}/ca.crt"
+    else
+        if [ -f "/etc/docker/certs.d/${registry}/ca.crt" ]; then
+            echo "Reusing existing /etc/docker/certs.d/${registry}/ca.crt"
+        else
+            sudo cp "${cert}" "/etc/docker/certs.d/${registry}/ca.crt"
+        fi
+    fi
+    set_mitm_cert "${cert}"
+    sudo systemctl daemon-reload
+    sudo systemctl restart docker
+    sudo systemctl status docker
+}
+
+# command to add mitm cert to local trust store
+set_mitm_cert() {
+    sudo cp "${1}" "/etc/pki/ca-trust/source/anchors/$(basename ${1}).crt"
+    sudo update-ca-trust extract
+}
+
+do_post_install() {
+    # Disable postfix since we don't need an MTA
+    sudo systemctl disable --now postfix
+}
+
+override_dhcp_dns() {
+    nameserver_list="${1}"
+    sudo sed -i -e 's/PEERDNS="yes"/PEERDNS="no"/' /etc/sysconfig/network-scripts/ifcfg-*
+    sudo sed -i -e '/DNS[0-9]=/d' /etc/sysconfig/network-scripts/ifcfg-*
+    sudo sed -i -e '/nameserver/d' /etc/resolv.conf
+    nsnumber=1
+    for nameserver in $(echo ${nameserver_list} | tr ',' ' '); do
+        echo "nameserver ${nameserver}" | append /etc/resolv.conf
+        for script in /etc/sysconfig/network-scripts/ifcfg-*; do
+            echo "DNS${nsnumber}=${nameserver}" | append "${script}"
+        done
+        nsnumber=$((nsnumber++))
+    done
+}

bootstrap_plugins/os-router.plugin.sh

@@ -33,22 +33,6 @@ detect_os() {
         os="$(uname -s) $(uname -r) $(cat /etc/*_ver* /etc/*-rel*)"
     fi
 
-    # Are we in a container?
-#    if ! [ -z "$(cut -d':' -f3 /proc/1/cgroup | grep '^/$')" ]; then
-#
-#        # Detect if we're running under Docker
-#        if ! [ -z "$(cut -d':' -f3 /proc/1/cgroup | grep '^/docker/.*$')" ]; then
-#            container_flag=true
-#            os="docker ${os}"
-#
-#        # Or perhaps LXC?
-#        elif ! [ -z "$(cut -d':' -f3 /proc/1/cgroup | grep '^/lxc/.*$')" ]; then
-#            container_flag=true
-#            os="lxc ${os}"
-#        fi
-#
-#    fi
-
     # string to lowercase
     os="${os,,}"
 }
@@ -58,12 +42,6 @@ route_os() {
 
     case "${os}" in
 
-#        centos\ linux\ release\ 7.2*)
-#            source ${plugins}/centos72.plugin.sh
-#            ;;
-#        centos\ linux\ release\ 7.3*)
-#            source ${plugins}/centos73.plugin.sh
-#            ;;
         centos\ linux\ release\ 7.4*)
             source ${plugins}/centos74.plugin.sh
             ;;
@@ -73,9 +51,9 @@ route_os() {
         centos\ linux\ release\ 7.6*)
             source ${plugins}/centos76.plugin.sh
             ;;
-#        dockerized\ centos\ linux\ release\ 7.2*)
-#            source ${plugins}/centos72-docker.plugin.sh
-#            ;;
+        centos\ linux\ release\ 7.7*)
+            source ${plugins}/centos77.plugin.sh
+            ;;
 
         # Die on unknowns
         *)

patches/3.3.0.2/Dockerfile

@@ -0,0 +1,27 @@
+# Fixes to the default 3.3.0.2 reduced image.
+
+# Build on object-reduced image (GA release)
+FROM emccorp/ecs-software-3.3.0:3.3.0.2
+
+# Fix disk partitioning script
+RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \
+    && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh
+
+# Set VNest useSeperateThreadPools to True
+#RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ <prop key="object.UseSeparateThreadPools">true</prop>' $f
+
+# Allow allocation of different blocks of a chunk to be stored on the same partition
+#RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>' $f || sed -i 's#<config:boolean name="allowAllocationOnIgnoredPartitions" value="false" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#g' /opt/storageos/conf/ssm-cf-conf.xml $f
+
+## Increase memory for transformsvc
+#RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc
+
+## Set memory for objcontrolsvc
+#RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc
+
+# Set georeceiver's initialBufferNumOnHeap to something smaller for CE
+#RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f ||  sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f
+#RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '<prop key="object.InitialBufferNumOnHeap">10</prop>' $f || sed -i 's#<prop key="object.InitialBufferNumOnHeap">80</prop>#<prop key="object.InitialBufferNumOnHeap">10</prop>#g' $f
+
+# Configure CM Object properties: Disable minimum storage device count
+#RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f ||  sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f

patches/3.3.0.2/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+source image.conf
+
+docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" .

patches/3.3.0.2/image.conf

@@ -0,0 +1,3 @@
+BASE_IMAGE="emcvipr/object:3.3.0.0-109089.4bf8a5a-reduced"
+IMAGE_REPO="emccorp/ecs-software-3.3.0"
+IMAGE_VERSION="3.3.0.2"

patches/libexec/patch_script.sh

@@ -51,10 +51,10 @@ done
 
 # Adjust memory adjustment
 sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc
-sed -i s/-Xmx2048m/-Xmx3072m/ /opt/storageos/bin/blobsvc
+sed -i s/-Xmx2688m/-Xmx3072m/ /opt/storageos/bin/blobsvc
 # sed -i s/-Xmx512m/-Xmx256m/ /opt/storageos/bin/metering
 sed -i s/\$\(_get_mem_fraction_mb\ 428.3m\ 0.0081\)/64/ /opt/storageos/bin/ecsportalsvc
-sed -i s/-Xmx384m/-Xmx768m/ /opt/storageos/bin/sr
+sed -i s/-Xmx640m/-Xmx768m/ /opt/storageos/bin/sr
 # sed -i s/-Xmx128m/-Xmx64m/ /opt/storageos/bin/dtquery
 sed -i s/-Xmx192m/-Xmx128m/ /opt/storageos/bin/eventsvc
 sed -i s/-Xmx256m/-Xmx128m/ /opt/storageos/bin/resourcesvc

release.conf

@@ -13,6 +13,8 @@
 #
 ### Older versions ( 3.3 )
 # release_artifact="emccorp/ecs-software-3.3.0"
+# release_tag="3.3.0.2"
+# release_tag="3.3.0.0"
 # release_tag="latest"
 #
 ### Older versions ( 3.2 )

ui/ansible/roles/CentOS_7_purge/tasks/main.yml

@@ -1,4 +1,4 @@
-#- name: CentOS 7 | Destroy GPT on ECS block devices
+k#- name: CentOS 7 | Destroy GPT on ECS block devices
 #  shell: echo -e 'x\nz\nY\nY\n' | gdisk {{ item }}
 #  with_items:
 #    - "{{ ecs_block_devices }}"

ui/ansible/roles/common_deploy/vars/main.yml

@@ -11,5 +11,6 @@ config_files:
   - /opt/storageos/bin/transformsvc
   - /opt/storageos/bin/objcontrolsvc
   - /opt/storageos/bin/sr
+  - /opt/storageos/bin/blobsvc
   - /opt/storageos/conf/georeceiver-conf.xml
   - /opt/storageos/conf/vnest-common-conf.xml

ui/etc/config.yml

@@ -32,7 +32,7 @@ ui:
   ffx_sem: /opt/ffx.sem
 product:
   name: ECS
-  version: 3.3.0.0
+  version: 3.3.0.2
   vendor: Dell EMC
   flavor: Community Edition
   slogan: Free and Frictionless

ui/etc/release.conf

@@ -8,10 +8,10 @@
 # it is provided by or on behalf of EMC.
 
 release_name="ECS Community Edition"
-release_version="3.3.0.0"
+release_version="3.3.0.2"
 release_product="ECS Software"
 release_artifact="emccorp/ecs-software-3.3.0"
-release_tag="3.3.0.0"
+release_tag="3.3.0.2"
 release_common_name="emccorp/ecs-software:latest"
 
 docker_host_root="/opt/emc/ecs-install"