[PLAT-13057] Universe Spec and Info structure refactor as a follow up…

… to D33904

Summary:
Continue addressing comments from D33904. https://phorge.dev.yugabyte.com/D33904#1068025 has a few good comments from Sanketh. Addressing them here as a follow-on change.

1. Introduced a universe level UniverseNetworkingSpec and moved `assign_public_ip`, `assign_static_public_ip`, `enable_ipv6` and the `communication_ports` from cluster level to universe level.
1. `CloudProviderSpec` of a cluster was a mix of provider and placement settings. Separated these into `ClusterProviderSpec` and `ClusterPlacementSpec`.
1. `ClusterPlacementSpec` was present as a read-only under UniverseInfo. But this should be editable by user. Added `ClusterPlacementSpec` to ClusterSpec that can optionally be specified by user when creating/editing the cluster.
1. Moved specifying the ybdb software version from cluster level to universe level. The same DB version will be maintained across all clusters of a universe.
1. Moved `use_time_sync` and `use_systemd` from cluster level to universe level.
1. Improved docs in ClusterStorageSpec
1. Defined default ports in CommunicationPortsSpec
1. Removed the `op_type` and instead kept the `encryptionAtRestEnabled` to identify the current encryption state of the Universe.
1. Removed all ports from NodeDetails as these are needed only for internal implementation. The user configures it only at the Universe level

Test Plan: unit test

Reviewers: sanketh, dshubin, amalyshev, skurapati, svarshney, #yba-api-review!

Reviewed By: amalyshev, skurapati, svarshney

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D34466

managed/build.sbt

@@ -832,9 +832,6 @@ lazy val javaGenV2Server = project.in(file("target/openapi"))
     openApiTemplateDir := (baseDirectory.value / resDir / "openapi_templates/").absolutePath,
     openApiValidateSpec := SettingDisabled,
     openApiGenerate := (openApiGenerate dependsOn openApiCopyIgnoreFile).value,
-    openApiTypeMappings := Map[String, String](
-      "OffsetDateTime" -> "java.util.Date"
-    ),
     // style plugin configurations
     openApiStyleSpec := baseDirectory.value / resDir / "openapi.yaml",
     openApiStyleConfig := Some(baseDirectory.value / resDir / "openapi_style_validator.conf"),

managed/client/java/openapi-java-config-v2.json

@@ -14,5 +14,6 @@
         "developerName": "Yugabyte",
         "developerEmail": "oss-maintainers@yugabyte.com",
         "licenseName": "POLYFORM-FREE-TRIAL-LICENSE-1.0.0",
-        "library": "jersey2"
+        "library": "jersey2",
+        "dateLibrary": "java8"
     }

managed/client/java/v2/pom.xml

@@ -143,8 +143,8 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>3.8.1</version>
                 <configuration>
-                        <source>1.7</source>
-                        <target>1.7</target>
+                        <source>1.8</source>
+                        <target>1.8</target>
                     <fork>true</fork>
                     <meminitial>128m</meminitial>
                     <maxmem>512m</maxmem>
@@ -168,7 +168,7 @@
                 </executions>
                 <configuration>
                     <doclint>none</doclint>
-                        <source>1.7</source>
+                        <source>1.8</source>
                     <tags>
                         <tag>
                             <name>http.response.details</name>
@@ -276,15 +276,9 @@
             <version>${jackson-databind-nullable-version}</version>
         </dependency>
         <dependency>
-            <groupId>com.github.joschi.jackson</groupId>
-            <artifactId>jackson-datatype-threetenbp</artifactId>
-            <version>${threetenbp-version}</version>
-        </dependency>
-        <!-- Base64 encoding that works in both JVM and Android -->
-        <dependency>
-            <groupId>com.brsanthu</groupId>
-            <artifactId>migbase64</artifactId>
-            <version>2.2</version>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+            <version>${jackson-version}</version>
         </dependency>
         <dependency>
             <groupId>javax.annotation</groupId>
@@ -312,7 +306,6 @@
         <jackson-version>2.10.5</jackson-version>
         <jackson-databind-version>2.10.5.1</jackson-databind-version>
         <jackson-databind-nullable-version>0.2.1</jackson-databind-nullable-version>
-        <threetenbp-version>2.9.10</threetenbp-version>
         <javax-annotation-version>1.3.2</javax-annotation-version>
         <junit-version>4.13.1</junit-version>
     </properties>

managed/src/main/java/api/v2/mappers/ClusterMapper.java

@@ -3,7 +3,10 @@
 
 import api.v2.models.ClusterInfo;
 import api.v2.models.ClusterSpec;
+import api.v2.models.PlacementAZ;
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster;
+import com.yugabyte.yw.models.helpers.PlacementInfo;
+import org.mapstruct.InheritInverseConfiguration;
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;
 import org.mapstruct.NullValueCheckStrategy;
@@ -16,6 +19,7 @@ public interface ClusterMapper {
   @Mapping(target = "storageSpec", source = "userIntent")
   @Mapping(target = "networkingSpec", source = "userIntent")
   @Mapping(target = "providerSpec", source = "userIntent")
+  @Mapping(target = "placementSpec", source = "placementInfo")
   @Mapping(target = "providerSpec.imageBundleUuid", source = "userIntent.imageBundleUUID")
   @Mapping(
       target = "providerSpec.provider",
@@ -25,10 +29,19 @@ public interface ClusterMapper {
   ClusterSpec toV2ClusterSpec(Cluster v1Cluster);
 
   @Mapping(target = "userIntent", source = ".")
+  @Mapping(target = "placementInfo", source = "placementSpec")
   // null check is required here to avoid overwriting the auto generated uuid with null
   @Mapping(target = "uuid", nullValueCheckStrategy = NullValueCheckStrategy.ALWAYS)
   Cluster toV1Cluster(ClusterSpec clusterSpec);
 
   @Mapping(target = ".", source = "userIntent")
   ClusterInfo toV2ClusterInfo(Cluster v1Cluster);
+
+  // used implicitly in above mapping
+  @Mapping(target = "numNodesInAZ", source = "numNodesInAz")
+  @Mapping(target = "isAffinitized", source = "leaderAffinity")
+  PlacementInfo.PlacementAZ toV1PlacementAZ(PlacementAZ placementAZ);
+
+  @InheritInverseConfiguration
+  PlacementAZ toV2PlacementAZ(PlacementInfo.PlacementAZ placementAZ);
 }

managed/src/main/java/api/v2/mappers/UniverseDefinitionTaskParamsDecorator.java

@@ -1,15 +1,18 @@
 // Copyright (c) YugaByte, Inc.
 package api.v2.mappers;
 
-import api.v2.models.EncryptionAtRestSpec;
-import api.v2.models.EncryptionAtRestSpec.OpTypeEnum;
+import api.v2.models.CloudSpecificInfo;
 import api.v2.models.EncryptionInTransitSpec;
 import api.v2.models.UniverseCreateSpec;
 import api.v2.models.UniverseCreateSpecYcql;
 import api.v2.models.UniverseCreateSpecYsql;
+import api.v2.models.UniverseNetworkingSpec;
+import api.v2.models.UniverseSpec;
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster;
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent;
+import java.util.ArrayList;
+import java.util.List;
 
 public abstract class UniverseDefinitionTaskParamsDecorator
     implements UniverseDefinitionTaskParamsMapper {
@@ -19,14 +22,34 @@ public UniverseDefinitionTaskParamsDecorator(UniverseDefinitionTaskParamsMapper
     this.delegate = delegate;
   }
 
+  // copy some of the details from v1 UniverseDefinitionTaskParams into UniverseSpec
+  private void updateUniverseSpec(
+      UniverseDefinitionTaskParams v1UniverseTaskParams, UniverseSpec universeSpec) {
+    // fetch name of universe from primary cluster and set it at top-level in v2
+    UserIntent primaryUserIntent = v1UniverseTaskParams.getPrimaryCluster().userIntent;
+    universeSpec.setName(primaryUserIntent.universeName);
+    // fetch yb software version from primary cluster and set it at top-level in v2
+    universeSpec.setYbSoftwareVersion(primaryUserIntent.ybSoftwareVersion);
+    // time sync
+    universeSpec.setUseTimeSync(primaryUserIntent.useTimeSync);
+    // systemd
+    universeSpec.setUseSystemd(primaryUserIntent.useSystemd);
+    // fetch networking spec from primary cluster and set it at top-level networking spec in v2
+    UniverseNetworkingSpec v2UnivNetworkingSpec = universeSpec.getNetworkingSpec();
+    v2UnivNetworkingSpec
+        .assignPublicIp(primaryUserIntent.assignPublicIP)
+        .assignStaticPublicIp(primaryUserIntent.assignStaticPublicIP)
+        .enableIpv6(primaryUserIntent.enableIPV6);
+  }
+
   @Override
   public UniverseCreateSpec toV2UniverseCreateSpec(
       UniverseDefinitionTaskParams v1UniverseTaskParams) {
     UniverseCreateSpec universeSpec = delegate.toV2UniverseCreateSpec(v1UniverseTaskParams);
-    // fetch name of universe from primary cluster and set it at top-level in v2
-    UserIntent primaryUserIntent = v1UniverseTaskParams.getPrimaryCluster().userIntent;
-    universeSpec.getSpec().setName(primaryUserIntent.universeName);
+    // update unmapped properties of spec within created universeSpec
+    updateUniverseSpec(v1UniverseTaskParams, universeSpec.getSpec());
     // fetch ysql/ycql password of universe from primary cluster and set it at top-level in v2
+    UserIntent primaryUserIntent = v1UniverseTaskParams.getPrimaryCluster().userIntent;
     if (primaryUserIntent.ysqlPassword != null) {
       universeSpec.ysql(new UniverseCreateSpecYsql().password(primaryUserIntent.ysqlPassword));
     }
@@ -37,45 +60,92 @@ public UniverseCreateSpec toV2UniverseCreateSpec(
   }
 
   @Override
-  public UniverseDefinitionTaskParams toV1UniverseDefinitionTaskParamsFromCreateSpec(
-      UniverseCreateSpec universeCreateSpec) {
-    UniverseDefinitionTaskParams params =
-        delegate.toV1UniverseDefinitionTaskParamsFromCreateSpec(universeCreateSpec);
+  public UniverseSpec toV2UniverseSpec(UniverseDefinitionTaskParams v1UniverseTaskParams) {
+    UniverseSpec universeSpec = delegate.toV2UniverseSpec(v1UniverseTaskParams);
+    updateUniverseSpec(v1UniverseTaskParams, universeSpec);
+    return universeSpec;
+  }
+
+  @Override
+  public UniverseDefinitionTaskParams toV1UniverseDefinitionTaskParams(UniverseSpec universeSpec) {
+    UniverseDefinitionTaskParams params = delegate.toV1UniverseDefinitionTaskParams(universeSpec);
     // set universeName, ysqlPassword, ycqlPassword into all cluster's userIntent
     if (params.clusters != null) {
       for (Cluster cluster : params.clusters) {
         if (cluster.userIntent != null) {
-          cluster.userIntent.universeName = universeCreateSpec.getSpec().getName();
-          if (universeCreateSpec.getYsql() != null) {
-            cluster.userIntent.ysqlPassword = universeCreateSpec.getYsql().getPassword();
-          }
-          if (universeCreateSpec.getYcql() != null) {
-            cluster.userIntent.ycqlPassword = universeCreateSpec.getYcql().getPassword();
-          }
+          cluster.userIntent.universeName = universeSpec.getName();
         }
       }
     }
-    // set encryptionAtRestConfig
-    if (universeCreateSpec.getSpec() != null
-        && universeCreateSpec.getSpec().getEncryptionAtRestSpec() != null) {
-      EncryptionAtRestSpec encryptionAtRestSpec =
-          universeCreateSpec.getSpec().getEncryptionAtRestSpec();
-      params.encryptionAtRestConfig = delegate.toV1EncryptionAtRestConfig(encryptionAtRestSpec);
-      params.encryptionAtRestConfig.encryptionAtRestEnabled =
-          encryptionAtRestSpec != null && encryptionAtRestSpec.getOpType() == OpTypeEnum.ENABLE;
+    UserIntent target = params.getPrimaryCluster().userIntent;
+    // set yb software version into primary cluster
+    if (universeSpec != null && universeSpec.getYbSoftwareVersion() != null) {
+      target.ybSoftwareVersion = universeSpec.getYbSoftwareVersion();
+    }
+    // use time sync
+    if (universeSpec != null && universeSpec.getUseTimeSync() != null) {
+      target.useTimeSync = universeSpec.getUseTimeSync();
     }
-    // set encryptionInTransit
-    if (universeCreateSpec.getSpec() != null
-        && universeCreateSpec.getSpec().getEncryptionInTransitSpec() != null) {
-      EncryptionInTransitSpec source = universeCreateSpec.getSpec().getEncryptionInTransitSpec();
-      UserIntent target = params.getPrimaryCluster().userIntent;
+    // use systemd
+    if (universeSpec != null && universeSpec.getUseSystemd() != null) {
+      target.useSystemd = universeSpec.getUseSystemd();
+    }
+    // set encryptionInTransit into primary cluster
+    if (universeSpec != null && universeSpec.getEncryptionInTransitSpec() != null) {
+      EncryptionInTransitSpec source = universeSpec.getEncryptionInTransitSpec();
       target.enableNodeToNodeEncrypt = source.getEnableNodeToNodeEncrypt();
       target.enableClientToNodeEncrypt = source.getEnableClientToNodeEncrypt();
       params.rootCA = source.getRootCa();
       params.setClientRootCA(source.getClientRootCa());
       params.rootAndClientRootCASame =
           source.getRootCa() != null && source.getRootCa().equals(source.getClientRootCa());
     }
+    // set networking into primary cluster
+    if (universeSpec != null && universeSpec.getNetworkingSpec() != null) {
+      UniverseNetworkingSpec source = universeSpec.getNetworkingSpec();
+      target.assignPublicIP = source.getAssignPublicIp();
+      target.assignStaticPublicIP = source.getAssignStaticPublicIp();
+      target.enableIPV6 = source.getEnableIpv6();
+    }
     return params;
   }
+
+  @Override
+  public UniverseDefinitionTaskParams toV1UniverseDefinitionTaskParamsFromCreateSpec(
+      UniverseCreateSpec universeCreateSpec) {
+    UniverseDefinitionTaskParams params =
+        toV1UniverseDefinitionTaskParams(universeCreateSpec.getSpec());
+    // set Arch
+    params.arch = toV1Arch(universeCreateSpec.getArch());
+    // set ysqlPassword, ycqlPassword into all cluster's userIntent
+    if (params.clusters != null) {
+      for (Cluster cluster : params.clusters) {
+        if (cluster.userIntent != null) {
+          if (universeCreateSpec.getYsql() != null) {
+            cluster.userIntent.ysqlPassword = universeCreateSpec.getYsql().getPassword();
+          }
+          if (universeCreateSpec.getYcql() != null) {
+            cluster.userIntent.ycqlPassword = universeCreateSpec.getYcql().getPassword();
+          }
+        }
+      }
+    }
+    return params;
+  }
+
+  // Need this due to a MapStruct limitation https://github.com/mapstruct/mapstruct/issues/3165
+  // Arrays cannot be mapped to collection. So doing it manually here for lunIndexes
+  @Override
+  public CloudSpecificInfo toV2CloudSpecificInfo(
+      com.yugabyte.yw.models.helpers.CloudSpecificInfo source) {
+    CloudSpecificInfo target = delegate.toV2CloudSpecificInfo(source);
+    if (source.lun_indexes != null) {
+      List<Integer> lunIdx = new ArrayList<>();
+      for (Integer lux : source.lun_indexes) {
+        lunIdx.add(lux);
+      }
+      target.setLunIndexes(lunIdx);
+    }
+    return target;
+  }
 }

managed/src/main/java/api/v2/mappers/UniverseDefinitionTaskParamsMapper.java

@@ -1,6 +1,8 @@
 // Copyright (c) YugaByte, Inc.
 package api.v2.mappers;
 
+import api.v2.models.CloudSpecificInfo;
+import api.v2.models.EncryptionAtRestInfo;
 import api.v2.models.EncryptionAtRestSpec;
 import api.v2.models.EncryptionInTransitSpec;
 import api.v2.models.NodeDetails;
@@ -19,6 +21,9 @@
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent;
 import com.yugabyte.yw.models.helpers.NodeDetails.MasterState;
 import com.yugabyte.yw.models.helpers.TaskType;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.Date;
 import org.mapstruct.DecoratedWith;
 import org.mapstruct.InheritInverseConfiguration;
 import org.mapstruct.Mapper;
@@ -69,20 +74,22 @@ public KubernetesGFlagsUpgradeParams toKubernetesGFlagsUpgradeParams(
   @Mapping(target = "overridePrebuiltAmiDbVersion", source = "overridePrebuiltAmiDBVersion")
   @Mapping(target = "encryptionAtRestSpec", source = "encryptionAtRestConfig")
   @Mapping(target = "encryptionInTransitSpec", source = ".")
+  @Mapping(target = "networkingSpec.communicationPorts", source = "communicationPorts")
   @Mapping(target = "ysql", source = ".")
   @Mapping(target = "ycql", source = ".")
   UniverseSpec toV2UniverseSpec(UniverseDefinitionTaskParams v1UniverseTaskParams);
 
-  @Mapping(target = ".", source = "spec")
-  @Mapping(target = ".", source = "arch")
-  UniverseDefinitionTaskParams toV1UniverseDefinitionTaskParamsFromCreateSpec(
-      UniverseCreateSpec universeCreateSpec);
-
   @InheritInverseConfiguration(name = "toV2UniverseSpec")
   UniverseDefinitionTaskParams toV1UniverseDefinitionTaskParams(UniverseSpec universeSpec);
 
+  // This mapping is overridden in the Decorator, and only that version is used.
+  @InheritInverseConfiguration(name = "toV2UniverseCreateSpec")
+  UniverseDefinitionTaskParams toV1UniverseDefinitionTaskParamsFromCreateSpec(
+      UniverseCreateSpec universeCreateSpec);
+
   @Mapping(target = "universeUuid", source = "universeUUID")
   @Mapping(target = "updatingTaskUuid", source = "updatingTaskUUID")
+  @Mapping(target = "encryptionAtRestInfo", source = "encryptionAtRestConfig")
   UniverseInfo toV2UniverseInfo(UniverseDefinitionTaskParams v1UniverseTaskParams);
 
   // below methods are used implicitly to generate other mapping
@@ -109,8 +116,18 @@ UniverseDefinitionTaskParams toV1UniverseDefinitionTaskParamsFromCreateSpec(
   EncryptionAtRestSpec toV2EncryptionAtRestSpec(EncryptionAtRestConfig v1EncryptionAtRestConfig);
 
   @InheritInverseConfiguration
+  @Mapping(target = "type", constant = "DATA_KEY")
+  @Mapping(
+      target = "opType",
+      expression =
+          "java(v2EncryptionAtRestSpec.getKmsConfigUuid() != null ?"
+              + " com.yugabyte.yw.forms.EncryptionAtRestConfig.OpType.ENABLE :"
+              + " com.yugabyte.yw.forms.EncryptionAtRestConfig.OpType.DISABLE)")
   EncryptionAtRestConfig toV1EncryptionAtRestConfig(EncryptionAtRestSpec v2EncryptionAtRestSpec);
 
+  @Mapping(target = "encryptionAtRestStatus", source = "encryptionAtRestEnabled")
+  EncryptionAtRestInfo toV2EncryptionAtRestInfo(EncryptionAtRestConfig v1EncryptionAtRestConfig);
+
   // copy EIT from primary cluster to top-level in v2
   default EncryptionInTransitSpec toV2EncryptionInTransitSpec(
       UniverseDefinitionTaskParams universeDetails) {
@@ -162,6 +179,20 @@ UniverseInfo.SoftwareUpgradeStateEnum mapSoftwareUpgradeState(
   @Mapping(target = "disksAreMountedByUuid", source = "disksAreMountedByUUID")
   NodeDetails toV2NodeDetails(com.yugabyte.yw.models.helpers.NodeDetails v1NodeDetails);
 
+  @Mapping(target = "assignPublicIp", source = "assignPublicIP")
+  @Mapping(target = "instanceType", source = "instance_type")
+  @Mapping(target = "mountRoots", source = "mount_roots")
+  @Mapping(target = "privateDns", source = "private_dns")
+  @Mapping(target = "privateIp", source = "private_ip")
+  @Mapping(target = "publicDns", source = "public_dns")
+  @Mapping(target = "publicIp", source = "public_ip")
+  @Mapping(target = "rootVolume", source = "root_volume")
+  @Mapping(target = "secondaryPrivateIp", source = "secondary_private_ip")
+  @Mapping(target = "secondarySubnetId", source = "secondary_subnet_id")
+  @Mapping(target = "subnetId", source = "subnet_id")
+  CloudSpecificInfo toV2CloudSpecificInfo(
+      com.yugabyte.yw.models.helpers.CloudSpecificInfo cloudSpecificInfo);
+
   @ValueMappings({
     @ValueMapping(target = "TOBEADDED", source = "ToBeAdded"),
     @ValueMapping(target = "INSTANCECREATED", source = "InstanceCreated"),
@@ -199,4 +230,11 @@ UniverseInfo.SoftwareUpgradeStateEnum mapSoftwareUpgradeState(
   })
   NodeDetails.StateEnum toV2NodeState(
       com.yugabyte.yw.models.helpers.NodeDetails.NodeState v1NodeState);
+
+  default OffsetDateTime toOffsetDateTime(Date date) {
+    if (date == null) {
+      return null;
+    }
+    return date.toInstant().atOffset(ZoneOffset.UTC);
+  }
 }

managed/src/main/java/api/v2/mappers/UniverseRespMapper.java

@@ -4,6 +4,7 @@
 import api.v2.models.UniverseResp;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.ZoneOffset;
 import org.mapstruct.DecoratedWith;
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;
@@ -25,7 +26,10 @@ public interface UniverseRespMapper {
   UniverseResp toV2UniverseResp(com.yugabyte.yw.forms.UniverseResp v1UniverseResp);
 
   // below methods are used implicitly to generate above mapping
-  default java.util.Date parseToOffsetDateTime(String datetime) throws ParseException {
-    return new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy").parse(datetime);
+  default java.time.OffsetDateTime parseToOffsetDateTime(String datetime) throws ParseException {
+    return new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy")
+        .parse(datetime)
+        .toInstant()
+        .atOffset(ZoneOffset.UTC);
   }
 }