Merge pull request #874 from globocom/dev

Dev > Master

dbaas/drivers/base.py

@@ -473,6 +473,9 @@ def get_final_infra_credentials(self):
             credential_type=self.credential_type
         )
         return credential.user, credential.password
+
+    def create_db_params_changes(self, database):
+        raise NotImplementedError('Not implemented for Driver')
 
 
 class DatabaseStatus(object):

dbaas/drivers/mysqldb.py

@@ -13,6 +13,7 @@
 from physical.models import Instance
 from drivers import BaseDriver, DatabaseInfraStatus, DatabaseStatus
 from drivers import errors as driver_errors
+from django.core.exceptions import ObjectDoesNotExist
 
 
 LOG = logging.getLogger(__name__)
@@ -537,7 +538,57 @@ def remove_metric_collector_user(self, username):
 
     def get_start_pty_default(self):
         return True
+
+    def create_db_params_changes(self, database):
+        from physical.models import DatabaseInfraParameter
 
+        parameter = self.get_max_connections_parameter()
+        value = self.get_max_connections_value(database)
+
+        LOG.info('Parameter {} will be set to {}'.format(parameter.name, value))
+
+        DatabaseInfraParameter.update_parameter_value(
+            databaseinfra=database.databaseinfra,
+            parameter=parameter,
+            value=value
+        )
+
+    def get_max_connections_parameter(self):
+        from physical.models import Parameter
+
+        param = Parameter.objects.filter(name='max_connections').first()
+        if not param:
+            raise ObjectDoesNotExist('Parametro max_connections nao foi encontrado')
+
+        return param
+
+    def get_max_connections_value(self, database):
+        from system.models import Configuration
+
+        steps = Configuration.get_by_name('max_connections_steps')
+        if not steps:
+            raise ObjectDoesNotExist('Configuration max_connections_steps nao foi encontrado')
+
+        offering = database.databaseinfra.offering
+        return self.get_max_connections_value_from_steps(steps, offering)
+
+
+    def get_max_connections_value_from_steps(self, steps, offering):
+        # steps ~= "0:1000,32768:2000,65536:4000"
+        steps_list = steps.split(',')
+        steps_map = {}
+
+        for step in steps_list:
+            # step ~= "0:1000"
+            steps_map[step.split(':')[0]] = step.split(':')[1]
+
+        ram = str(offering.memory_size_mb)
+
+        # steps_map ~= {'0': '1000', '32768': '2000'...}
+        if ram not in steps_map:
+            return int(steps_map['0'])
+
+        return int(steps_map[ram])
 
 class MySQLFOXHA(MySQL):
 

dbaas/drivers/replication_topologies/base.py

@@ -735,6 +735,9 @@ def get_stop_database_vm_steps(self):
 
     def get_auto_upgrade_database_vm_offering(self):
         raise NotImplementedError('Not implemented for topology')
+
+    def get_configure_db_params_steps(self):
+        raise NotImplementedError('Not implemented for topology')
 
     def get_start_database_vm_steps(self):
         return [{

dbaas/drivers/replication_topologies/mongodb.py

@@ -916,6 +916,101 @@ def get_recreate_slave_steps(self):
                 'workflow.steps.util.database.ConfigurePrometheusMonitoring'
             )
         }]
+
+    def get_auto_upgrade_database_vm_offering(self):
+        return [{
+            'Create new VM': (
+                'workflow.steps.util.infra.OfferingAutoUpgrade',
+                'workflow.steps.util.host_provider.AllocateIPTemporaryInstance',
+                'workflow.steps.util.host_provider.CreateVirtualMachineTemporaryInstance',
+                # 'workflow.steps.util.dns.CreateDNS',
+                # 'workflow.steps.util.dns.CheckIsReady',
+                'workflow.steps.util.vm.WaitingBeReadyTemporaryInstance',
+                'workflow.steps.util.vm.UpdateOSDescriptionTemporaryInstance',
+            )}, {
+            'Attach new Volume': (
+                'workflow.steps.util.volume_provider.NewVolumeFromSnapshot',
+                'workflow.steps.util.volume_provider.AttachDataVolumeTemporaryInstance',
+                'workflow.steps.util.volume_provider.MountDataVolumeTemporaryInstance',
+            )}, {
+            'Set SSL': (
+                'workflow.steps.util.ssl.UpdateOpenSSlLibIfConfiguredTemporaryInstance',
+                'workflow.steps.util.ssl.MongoDBUpdateCertificatesIfConfiguredTemporaryInstance',
+                'workflow.steps.util.ssl.CreateSSLFolderRollbackIfRunningIfConfiguredTemporaryInstance',
+                'workflow.steps.util.ssl.MongoDBCreateSSLConfForInfraIfConfiguredTemporaryInstance',
+                'workflow.steps.util.ssl.RequestSSLForInfraIfConfiguredTemporaryInstance',
+                'workflow.steps.util.ssl.CreateJsonRequestFileInfraIfConfiguredTemporaryInstance',
+                'workflow.steps.util.ssl.CreateCertificateInfraMongoDBIfConfiguredTemporaryInstance',
+                'workflow.steps.util.ssl.SetSSLFilesAccessMongoDBIfConfiguredTemporaryInstance',
+                'workflow.steps.util.ssl.UpdateExpireAtDateTemporaryInstance',
+            )}, {
+            'Configure Plan': (
+                'workflow.steps.util.plan.ConfigureTemporaryInstance',
+                'workflow.steps.util.plan.ConfigureLogTemporaryInstance',
+                'workflow.steps.util.metric_collector.ConfigureTelegrafTemporaryInstance',
+                'workflow.steps.util.database_upgrade_patch.MongoDBCHGBinStepTemporaryInstance',
+                'workflow.steps.util.database.StartDatabaseTemporaryInstance',
+                'workflow.steps.util.database.StartCheckOnlyOsProcessTemporaryInstance',
+                'workflow.steps.util.database.CheckIsUpTemporaryInstance',
+            )}, {
+            'Add Instance to ReplicaSet': (
+                'workflow.steps.mongodb.database.AddInstanceToReplicaSetTemporaryInstance',
+                'workflow.steps.util.database.WaitForReplicationTemporaryInstance',
+            )}, {
+            'Restart Telegraf and Rsyslog': (
+                'workflow.steps.util.metric_collector.RestartTelegrafTemporaryInstance',
+                'workflow.steps.util.database.StartRsyslogTemporaryInstance',
+            )}, {
+            'Replicate ACLs': (
+                'workflow.steps.util.acl.ReplicateAcls2NewInstanceTemporaryInstance',
+                'workflow.steps.util.acl.BindNewInstanceTemporaryInstance',
+            )}, {
+            'Add Alarms and Monitoring': (
+                'workflow.steps.util.zabbix.CreateAlarmsTemporaryInstance',
+                'workflow.steps.util.db_monitor.CreateMonitoringTemporaryInstance',
+                'workflow.steps.util.database.ConfigurePrometheusMonitoringTemporaryInstance',
+            )}, {
+            'Changing Primary': (
+                'workflow.steps.util.vm.ChangeMasterTemporaryInstance',
+            )}, {
+            'Resizing database': (
+                'workflow.steps.util.zabbix.DisableAlarms',
+                'workflow.steps.util.database.StopSlave',
+                'workflow.steps.util.database.StopRsyslog',
+                'workflow.steps.util.database.Stop',
+                'workflow.steps.util.plan.ResizeConfigure',
+                'workflow.steps.util.plan.ConfigureLog',
+                'workflow.steps.util.host_provider.Stop',
+                'workflow.steps.util.host_provider.ChangeOffering',
+                'workflow.steps.util.host_provider.Start',
+                'workflow.steps.util.vm.WaitingBeReady',
+                'workflow.steps.util.database.Start',
+                'workflow.steps.util.database.StartSlave',
+                'workflow.steps.util.database.CheckIsUp',
+                'workflow.steps.util.database.WaitForReplication',
+                'workflow.steps.util.infra.Offering',
+                'workflow.steps.util.vm.InstanceIsSlave',
+                'workflow.steps.util.zabbix.EnableAlarms',
+            )}, {
+            'Returning Primary to original VM': (
+                'workflow.steps.util.vm.ChangeMasterNotTemporaryInstance',
+            )}, {
+            'Destroying temporary VM': (
+                'workflow.steps.util.db_monitor.DisableMonitoringTemporaryInstance',
+                'workflow.steps.util.zabbix.DestroyAlarmsTemporaryInstance',
+
+                'workflow.steps.util.database.StopRsyslogTemporaryInstance',
+                'workflow.steps.mongodb.database.RemoveInstanceFromReplicaSetTemporaryInstance',
+                'workflow.steps.util.database.StopTemporaryInstance',
+
+                'workflow.steps.util.volume_provider.DetachDataVolumeTemporaryInstance',
+                'workflow.steps.util.volume_provider.DestroyVolumeTemporaryInstance',
+
+                # 'workflow.steps.util.dns.DestroyDNSTemporaryInstance',
+                'workflow.steps.util.host_provider.DestroyVirtualMachineTemporaryInstance',
+                'workflow.steps.util.host_provider.DestroyIPTemporaryInstance',
+            )}
+        ]
 
     def get_host_migrate_steps(self):
         return [{

dbaas/drivers/replication_topologies/mysql.py

@@ -210,6 +210,15 @@ def get_deploy_steps(self):
                 'workflow.steps.util.database.MakeSnapshot',
             )
         }]
+
+    def get_configure_db_params_steps(self):
+        return [{
+            'Configuring DB Params': (
+                'workflow.steps.util.database.CreateParameterChange',
+                'workflow.steps.util.plan.ConfigureOnlyDBConfigFile',
+                'workflow.steps.util.database.ChangeDynamicParameters',
+                'workflow.steps.util.database.UpdateKernelParameters',
+            )}] + self.get_change_parameter_steps_final()
 
     def get_host_migrate_steps(self):
         return [{
@@ -553,10 +562,10 @@ def get_deploy_steps(self):
                 'workflow.steps.util.ssl.UpdateExpireAtDate',
             )}, {
             'Starting database': (
-                'workflow.steps.util.plan.ConfigureForNewInfra',
-                'workflow.steps.util.plan.ConfigureLogForNewInfra',
+                'workflow.steps.util.plan.ConfigureForNewInfra', # mysql_foxha_57_configuration.sh
+                'workflow.steps.util.plan.ConfigureLogForNewInfra', # rsyslog_config.sh
                 'workflow.steps.util.metric_collector.ConfigureTelegraf',
-                'workflow.steps.util.database.Start',
+                'workflow.steps.util.database.Start', # start database
                 'workflow.steps.util.metric_collector.RestartTelegraf',
                 'workflow.steps.util.database.StartRsyslog',
                 'workflow.steps.util.database.CheckIsUp',
@@ -1065,30 +1074,22 @@ def get_update_ssl_steps(self):
             'Disable monitoring and alarms': (
                 'workflow.steps.util.zabbix.DisableAlarms',
                 'workflow.steps.util.db_monitor.DisableMonitoring',
+                'workflow.steps.util.ssl.UpdateExpireAtDateRollback',
+                'workflow.steps.util.ssl.BackupSSLFolder',
             ),
         }] + [{
             'Disable SSL': (
                 'workflow.steps.util.ssl.UnSetReplicationUserRequireSSL',
             ),
         }] + [{
             'Configure SSL': (
-                'workflow.steps.util.ssl.UpdateExpireAtDateRollback',
-                'workflow.steps.util.ssl.MoveSSLFolder',
-                'workflow.steps.util.ssl.UpdateOpenSSlLib',
-                'workflow.steps.util.ssl.CreateSSLFolder',
-                'workflow.steps.util.ssl.CreateSSLConfForInfraEndPoint',
-                'workflow.steps.util.ssl.CreateSSLConfForInstanceIP',
-                'workflow.steps.util.ssl.RequestSSLForInfra',
-                'workflow.steps.util.ssl.RequestSSLForInstance',
+                'workflow.steps.util.ssl.UpdateSSLForInfra',
+                'workflow.steps.util.ssl.UpdateSSLForInstance',
                 'workflow.steps.util.ssl.CreateJsonRequestFileInfra',
                 'workflow.steps.util.ssl.CreateJsonRequestFileInstance',
                 'workflow.steps.util.ssl.CreateCertificateInfra',
                 'workflow.steps.util.ssl.CreateCertificateInstance',
                 'workflow.steps.util.ssl.SetSSLFilesAccessMySQL',
-                'workflow.steps.util.ssl.SetInfraConfiguredSSL',
-                'workflow.steps.util.plan.Configure',
-                'workflow.steps.util.plan.ConfigureLog',
-                'workflow.steps.util.metric_collector.ConfigureTelegraf',
                 'workflow.steps.util.ssl.UpdateExpireAtDate',
             ),
         }] + [{
@@ -1097,7 +1098,7 @@ def get_update_ssl_steps(self):
                 'workflow.steps.util.vm.ChangeMaster',
                 'workflow.steps.util.database.CheckIfSwitchMaster',
                 'workflow.steps.util.database.Stop',
-                # 'workflow.steps.util.ssl.RestoreSSLFolder4Rollback',
+                'workflow.steps.util.ssl.RestoreSSLFolder4Rollback',
                 'workflow.steps.util.database.Start',
                 'workflow.steps.util.metric_collector.RestartTelegraf',
                 'workflow.steps.util.database.CheckIfSwitchMasterRollback',
@@ -1111,7 +1112,6 @@ def get_update_ssl_steps(self):
             ),
         }] + [{
             'Enabling monitoring and alarms': (
-                'workflow.steps.util.db_monitor.UpdateInfraSSLMonitor',
                 'workflow.steps.util.db_monitor.EnableMonitoring',
                 'workflow.steps.util.zabbix.EnableAlarms',
             ),

dbaas/logical/templates/logical/database/details/parameters_tab.html

@@ -5,10 +5,12 @@
 <fieldset class="module aligned ">
   <div class="panel-heading">
     <h3 class="panel-title">Custom Parameters</h3>
+    {% if show_auto_configure_btn %}
+      <a href="{% url 'auto_configure_db_params_btn' database.id %}" target="_blank" class="btn btn-primary">AutoConfigure DB Params</a>
+    {% endif %}
   </div>
 
   <div class="panel-body">
-
     <table id="table-parameters" class="table table-striped table-hover" data-database-id="{{database.pk}}" >
       <thead>
         <tr>

dbaas/logical/urls.py

@@ -1,7 +1,7 @@
 from django.conf.urls import patterns, url
 from .views import refresh_status, toggle_monitoring, start_database_vm, stop_database_vm, set_attention, funct_send_all_chg
 from .views import (CredentialView, CredentialSSLView,
-                    credential_parameter_by_name, check_offering_sizes, resize_vm_from_btn)
+                    credential_parameter_by_name, check_offering_sizes, resize_vm_from_btn, auto_configure_db_params_btn)
 
 
 urlpatterns = patterns(
@@ -38,5 +38,8 @@
         name="check_offering_sizes"),
     url(r"^resize_vm/(?P<database_id>\d*)/(?P<resize_target>[\w\-\_]+)",
         resize_vm_from_btn,
-        name="resize_vm_from_btn")
+        name="resize_vm_from_btn"),
+    url(r"^autoconfigure_db_params/(?P<database_id>\d*[\w\-\_]+)",
+        auto_configure_db_params_btn,
+        name="auto_configure_db_params_btn")
 )

dbaas/logical/views.py

@@ -693,6 +693,17 @@ def get_context_data(self, **kwargs):
         self.context['form_status'] = self.form_status
         self.context['last_change_parameters'] = last_change_parameters
 
+        user_teams = self.request.user.team_set.all()
+        teams_names = []
+        for team in user_teams:
+            teams_names.append(team.name)
+
+        show_auto_configure_btn = False
+        if self.request.user.is_superuser or 'dbaas' in teams_names:
+            show_auto_configure_btn = True
+
+        self.context['show_auto_configure_btn'] = show_auto_configure_btn
+
         return self.context
 
     def post(self, request, *args, **kwargs):
@@ -2747,8 +2758,25 @@ def resize_vm_from_btn(request, database_id, resize_target):
     LOG.info("Starting database auto upgrade: database {}, user: {}".format(database, user))
 
     TaskRegister.auto_upgrade_database_vm_offering(
-        database=database, user=user
+        database=database, user=user, resize_target=resize_target
     )
 
     future_offering = database.get_future_offering(resize_target)
     return HttpResponse(json.dumps({'future_offering': future_offering.name}), content_type="application/json")
+
+
+@login_required()
+@method_decorator(csrf_exempt)
+def auto_configure_db_params_btn(request, database_id):
+    database = get_object_or_404(Database, pk=database_id)
+    user = request.user
+
+    from notification.tasks import TaskRegister
+
+    LOG.info("Starting auto configure DB Params: database {}, user: {}".format(database, user))
+
+    TaskRegister.configure_db_params(
+        database=database, user=user
+    )
+
+    return HttpResponse(({'success'}), content_type="application/json")

dbaas/maintenance/admin/__init__.py

@@ -29,6 +29,7 @@
 from . database_start_database_vm import DatabaseStartDatabaseVMAdmin
 from . database_stop_database_vm import DatabaseStopDatabaseVMAdmin
 from . database_auto_upgrade_vm_offering import DatabaseAutoUpgradeVMOferringAdmin
+from . database_configure_db_params import DatabaseConfigureDBParamsAdmin
 
 
 admin.site.register(models.Maintenance, MaintenanceAdmin)
@@ -79,3 +80,7 @@
 admin.site.register(
     models.DatabaseAutoUpgradeVMOffering, DatabaseAutoUpgradeVMOferringAdmin
 )
+
+admin.site.register(
+    models.DatabaseConfigureDBParams, DatabaseConfigureDBParamsAdmin
+)

dbaas/maintenance/admin/database_auto_upgrade_vm_offering.py

@@ -22,9 +22,11 @@ class DatabaseAutoUpgradeVMOferringAdmin(DatabaseMaintenanceTaskAdmin):
     )
 
     readonly_fields = (
-        "database", "task",
+        "database", "task", "resize_target",
         "started_at", "link_task", "finished_at", "status",
-        "maintenance_action", "task_schedule"
+        "maintenance_action", "task_schedule",
+        "number_of_instances", "number_of_instances_before",
+        "base_snapshot"
     )
 
     def maintenance_action(self, maintenance):
@@ -84,7 +86,8 @@ def retry_view(self, request, auto_upgrade_vm_offering_id):
         TaskRegister.auto_upgrade_database_vm_offering(
             database=retry_from.database,
             user=request.user,
-            retry_from=retry_from
+            retry_from=retry_from,
+            resize_target=retry_from.resize_target
         )
 
         url = reverse('admin:notification_taskhistory_changelist')