Resolve periodic SSM, CM, and dtquery crashes

Migrate ECS config changes into Ansible Took 2 hours 33 minutes
EMCECS · Jun 6, 2018 · 338d99e · 338d99e
1 parent f317406
commit 338d99e
Show file tree

Hide file tree

Showing 4 changed files with 103 additions and 47 deletions.
diff --git a/patches/3.2.0.1/Dockerfile b/patches/3.2.0.1/Dockerfile
@@ -3,27 +3,25 @@
 # Build on object-reduced image (GA release)
 FROM emcvipr/object:3.2.0.1-101633.6533e6c-reduced
 
-# Increase memory for transformsvc
-RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc
+# Fix disk partitioning script
+RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \
+    && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh
 
-# Set memory for objcontrolsvc
-RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc
+# Set VNest useSeperateThreadPools to True
+#RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ <prop key="object.UseSeparateThreadPools">true</prop>' $f
 
-# Fix disk partitioning script
-RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh
+# Allow allocation of different blocks of a chunk to be stored on the same partition
+#RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>' $f || sed -i 's#<config:boolean name="allowAllocationOnIgnoredPartitions" value="false" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#g' /opt/storageos/conf/ssm-cf-conf.xml $f
 
-RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh
+## Increase memory for transformsvc
+#RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc
 
-# Set VNest useSeperateThreadPools to True
-RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ <prop key="object.UseSeparateThreadPools">true</prop>' $f
+## Set memory for objcontrolsvc
+#RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc
 
 # Set georeceiver's initialBufferNumOnHeap to something smaller for CE
-RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f ||  sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f
-
-RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '<prop key="object.InitialBufferNumOnHeap">10</prop>' $f || sed -i 's#<prop key="object.InitialBufferNumOnHeap">80</prop>#<prop key="object.InitialBufferNumOnHeap">10</prop>#g' $f
+#RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f ||  sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f
+#RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '<prop key="object.InitialBufferNumOnHeap">10</prop>' $f || sed -i 's#<prop key="object.InitialBufferNumOnHeap">80</prop>#<prop key="object.InitialBufferNumOnHeap">10</prop>#g' $f
 
 # Configure CM Object properties: Disable minimum storage device count
-RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f ||  sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f
-
-# Allow allocation of different blocks of a chunk to be stored on the same partition
-RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>' $f || sed -i 's#<config:boolean name="allowAllocationOnIgnoredPartitions" value="false" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#g' /opt/storageos/conf/ssm-cf-conf.xml $f
+#RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f ||  sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f
diff --git a/ui/ansible/roles/common_baseline_install/tasks/main.yml b/ui/ansible/roles/common_baseline_install/tasks/main.yml
@@ -90,7 +90,7 @@
   loop_control:
     label: "/ecs/uuid-{{ item.0.stdout }}"
 
-- name: Common | Preallocate XFS extents for chunk storage
+- name: Common | Create blockbins on partitions
   script: additional_prep.sh {{ item }}1
   with_items: "{{ ecs_block_devices }}"
 

diff --git a/ui/ansible/roles/common_deploy/tasks/main.yml b/ui/ansible/roles/common_deploy/tasks/main.yml
@@ -12,6 +12,7 @@
     restart_policy: no
     volumes:
       - /opt/storageos/conf
+      - /opt/storageos/bin
       - /opt/storageos/ecsportal/conf
       - /etc
     entrypoint: /bin/echo "ECS configuration data container"
@@ -30,95 +31,146 @@
 #  Installer should support installing a "full" build on systems with 64GB RAM+ #74
 
 - name: Common | Copy configurations out of configuration data container
-  command: docker cp ecs-config:/opt/storageos/conf/{{item}} /host/{{item}}
+  command: docker cp ecs-config:{{item}} /host/{{item.split('/')[-1]}}
   with_items: "{{config_files}}"
   loop_control:
     label: "{{item}}"
 #    - dest: /host/application.conf
 #      src: /opt/storageos/ecsportal/conf/application.conf
 
-## Perform properties edits
+### Perform properties edits
 
-## SSM Object Properties
-# object.freeBlocksHighWatermarkLevels
+## ssm configuration
 - name: 'Common | Configure SSM Object properties: FBL-Highwater'
   lineinfile:
     dest: /host/ssm.object.properties
     state: present
     regexp: '^object\.freeBlocksHighWatermarkLevels=[0-9]*,[0-9]*'
     line: 'object.freeBlocksHighWatermarkLevels={{ "100,50" if ansible_local.data_node.ecs_block_size|int < 1000000000000 else "1000,200" }}'
-
-# object.freeBlocksLowWatermarkLevels
 - name: 'Common | Configure SSM Object properties: FBL-Lowwater'
   lineinfile:
     dest: /host/ssm.object.properties
     state: present
     regexp: '^object\.freeBlocksLowWatermarkLevels=[0-9]*,[0-9]*'
     line: 'object.freeBlocksLowWatermarkLevels={{ "0,20" if ansible_local.data_node.ecs_block_size|int < 1000000000000 else "0,100" }}'
+- name: 'Common | Configure Common Object properties: allowAllocationOnIgnoredPartitions'
+  lineinfile:
+    dest: /host/ssm-cf-conf.xml
+    state: present
+    backrefs: yes
+    regexp: '^(.*)name="allowAllocationOnIgnoredPartitions" value="false"(.*)'
+    line: '\1name="allowAllocationOnIgnoredPartitions" value="true"\2'
 
 ## Common Object Properties
 # When NumDtPerCoS is set to 128 (>=64GB RAM) then the number of btree
 # and journal chunks for write quadruples. This means when node count is
 # less than ~4 and total storage space is less than ~4TB, 128 NumDtPerCoS
 # can become unwieldy. This templated param should take into account node
 # count and total storage space available. https://github.com/EMCECS/ECS-CommunityEdition/issues/301
-
-# object.NumDirectoriesPerCoSForSystemDT
 - name: 'Common | Configure Common Object properties: System NumDtPerCoS'
   lineinfile:
     dest: /host/common.object.properties
     state: present
     regexp: '^object\.NumDirectoriesPerCoSForSystemDT=[0-9]*'
     line: 'object.NumDirectoriesPerCoSForSystemDT={{ "128" if ( ansible_memtotal_mb|int > 65536 and ansible_local.data_node.ecs_block_size|int > 1000000000000 and num_data_nodes|int > 3 ) else "32" }}'
-
-# object.NumDirectoriesPerCoSForUserDT
 - name: 'Common | Configure Common Object properties: User NumDtPerCoS'
   lineinfile:
     dest: /host/common.object.properties
     state: present
     regexp: '^object\.NumDirectoriesPerCoSForUserDT=[0-9]*'
     line: 'object.NumDirectoriesPerCoSForUserDT={{ "128" if ( ansible_memtotal_mb|int > 65536 and ansible_local.data_node.ecs_block_size|int > 1000000000000 and num_data_nodes|int > 3 ) else "32" }}'
 
-## CM Object Properties
+## cm configuration
 - name: 'Common | Configure CM Object properties: Disable Minimum Node Count'
   lineinfile:
     dest: /host/cm.object.properties
     state: present
-    regexp: '^object\.MustHaveEnoughResources=true'
+    regexp: 'object.MustHaveEnoughResources=true'
     line: 'object.MustHaveEnoughResources=false'
-
-## CM configuration
 - name: 'Common | Configure CM Read Page Cache: Max'
   lineinfile:
     dest: /host/cm-conf.xml
     state: present
-    regexp: '<prop key="object.BPlusTreeReaderOnHeapCacheV2MemoryMaxSize">536870912</prop>'
-    line: '<prop key="object.BPlusTreeReaderOnHeapCacheV2MemoryMaxSize">52428800</prop>'
-
+    backrefs: yes
+    regexp: '^(.*)key="object.BPlusTreeReaderOnHeapCacheV2MemoryMaxSize">536870912(.*)'
+    line: '\1key="object.BPlusTreeReaderOnHeapCacheV2MemoryMaxSize">52428800\2'
 - name: 'Common | Configure CM Read Page Cache: Core'
   lineinfile:
     dest: /host/cm-conf.xml
     state: present
-    regexp: '<prop key="object.BPlusTreeReaderOnHeapCacheV2MemoryCoreSize">419430400</prop>'
-    line: '<prop key="object.BPlusTreeReaderOnHeapCacheV2MemoryCoreSize">41943040</prop>'
-
+    backrefs: yes
+    regexp: '^(.*)key="object.BPlusTreeReaderOnHeapCacheV2MemoryCoreSize">419430400(.*)'
+    line: '\1key="object.BPlusTreeReaderOnHeapCacheV2MemoryCoreSize">41943040\2'
 - name: 'Common | Configure CM DT Write IO: OnHeap Buffers'
   lineinfile:
     dest: /host/cm-conf.xml
     state: present
-    regexp: '<constructor-arg name="initialBufferNumOnHeap" value="10"/>'
-    line: '<constructor-arg name="initialBufferNumOnHeap" value="4"/>'
-
+    backrefs: yes
+    regexp: '^(.*)name="initialBufferNumOnHeap" value="10"(.*)'
+    line: '\1name="initialBufferNumOnHeap" value="4"\2'
 - name: 'Common | Configure CM Geo Shipping IO: OnHeap Buffers'
   lineinfile:
     dest: /host/cm-conf.xml
     state: present
-    regexp: '<prop key="object.InitialBufferNumOnHeap">50</prop'
-    line: '<prop key="object.InitialBufferNumOnHeap">4</prop>'
+    backrefs: yes
+    regexp: '^(.*)key="object.InitialBufferNumOnHeap">50(.*)'
+    line: '\1key="object.InitialBufferNumOnHeap">4\2'
+
+## georeceiver configuration
+# Set georeceiver's initialBufferNumOnHeap to something smaller for CE
+- name: 'Common | Configure georeceiver initialBufferNumOnHeap (1/2)'
+  lineinfile:
+    dest: /host/georeceiver-conf.xml
+    state: present
+    backrefs: yes
+    regexp: '^(.*)name="initialBufferNumOnHeap" value="40"(.*)'
+    line: '\1name="initialBufferNumOnHeap" value="5"\2'
+- name: 'Common | Configure georeceiver initialBufferNumOnHeap (2/2)'
+  lineinfile:
+    dest: /host/georeceiver-conf.xml
+    state: present
+    backrefs: yes
+    regexp: '^(.*)key="object.InitialBufferNumOnHeap">80(.*)'
+    line: '\1key="object.InitialBufferNumOnHeap">10\2'
+
+## dtquery configuration
+- name: 'Common | Increase memory for dtquery'
+  lineinfile:
+    dest: /host/dtquery
+    state: present
+    backrefs: yes
+    regexp: '^(.*)Xmx128m(.*)$'
+    line: '\1Xmx256m\2'
+
+## transformsvc configuration
+- name: 'Common | Increase memory for transformsvc'
+  lineinfile:
+    dest: /host/transformsvc
+    state: present
+    backrefs: yes
+    regexp: '^(.*)Xmx128m(.*)$'
+    line: '\1Xmx512m\2'
+
+## objcontrolsvc configuration
+- name: 'Common | Increase memory for objcontrolsvc'
+  lineinfile:
+    dest: /host/objcontrolsvc
+    state: present
+    backrefs: yes
+    regexp: '^(.*)Xmx96m(.*)$'
+    line: '\1Xmx256m\2'
+
+## vnest configuration
+- name: 'Common | vnest enable UseSeparateThreadPools'
+  lineinfile:
+    dest: /host/vnest-common-conf.xml
+    state: present
+    insertafter: '<util:properties id="serviceProperties" location="classpath:systemsvc.object.properties" local-override="true">'
+    line: '        <prop key="object.UseSeparateThreadPools">true</prop>'
 
 ### Then docker cp them into the configuration container
 - name: Common | Merge configurations into configuration data container
-  command: docker cp /host/{{item}} ecs-config:/opt/storageos/conf/{{item}}
+  command: docker cp /host/{{item.split('/')[-1]}} ecs-config:{{item}}
   with_items: "{{config_files}}"
   loop_control:
     label: "{{item}}"

diff --git a/ui/ansible/roles/common_deploy/vars/main.yml b/ui/ansible/roles/common_deploy/vars/main.yml
@@ -2,7 +2,13 @@ ecs_environnment:
   SS_GENCONFIG: 1
 data_container: ecs-config
 config_files:
-  - ssm.object.properties
-  - common.object.properties
-  - cm.object.properties
-  - cm-conf.xml
+  - /opt/storageos/conf/ssm.object.properties
+  - /opt/storageos/conf/ssm-cf-conf.xml
+  - /opt/storageos/conf/common.object.properties
+  - /opt/storageos/conf/cm.object.properties
+  - /opt/storageos/conf/cm-conf.xml
+  - /opt/storageos/bin/dtquery
+  - /opt/storageos/bin/transformsvc
+  - /opt/storageos/bin/objcontrolsvc
+  - /opt/storageos/conf/georeceiver-conf.xml
+  - /opt/storageos/conf/vnest-common-conf.xml