[SW-2646] Calculate Metrics on Arbitrary Dataset

api-generation/src/main/scala/ai/h2o/sparkling/api/generation/common/MetricsConfigurations.scala

@@ -30,7 +30,7 @@ trait MetricsConfigurations {
         "The class makes available all metrics that shared across all algorithms, and ML problems." +
           " (classification, regression, dimension reduction)."),
       ModelMetricsSubstitutionContext(
-        "H2OBinomialMetrics",
+        "H2OBinomialMetricsBase",
         classOf[ModelMetricsBinomialV3[_, _]],
         Seq("H2OCommonMetrics"),
         "The class makes available all metrics that shared across all algorithms supporting binomial classification."),
@@ -40,7 +40,7 @@ trait MetricsConfigurations {
         Seq("H2OBinomialMetrics", "H2OGLMMetrics"),
         "The class makes available all binomial metrics supported by GLM algorithm."),
       ModelMetricsSubstitutionContext(
-        "H2ORegressionMetrics",
+        "H2ORegressionMetricsBase",
         classOf[ModelMetricsRegressionV3[_, _]],
         Seq("H2OCommonMetrics"),
         "The class makes available all metrics that shared across all algorithms supporting regression."),
@@ -55,7 +55,7 @@ trait MetricsConfigurations {
         Seq("H2ORegressionMetrics"),
         "The class makes available all regression metrics supported by CoxPH algorithm."),
       ModelMetricsSubstitutionContext(
-        "H2OMultinomialMetrics",
+        "H2OMultinomialMetricsBase",
         classOf[ModelMetricsMultinomialV3[_, _]],
         Seq("H2OCommonMetrics"),
         "The class makes available all metrics that shared across all algorithms supporting multinomial classification."),

api-generation/src/main/scala/ai/h2o/sparkling/api/generation/python/MetricsFactoryTemplate.scala

@@ -22,7 +22,7 @@ import ai.h2o.sparkling.api.generation.common.{EntitySubstitutionContext, ModelM
 object MetricsFactoryTemplate extends ((Seq[ModelMetricsSubstitutionContext]) => String) with PythonEntityTemplate {
 
   def apply(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
-    val metricClasses = metricSubstitutionContexts.map(_.entityName)
+    val metricClasses = getEntityNames(metricSubstitutionContexts)
     val imports = Seq("py4j.java_gateway.JavaObject") ++
       metricClasses.map(metricClass => s"ai.h2o.sparkling.ml.metrics.$metricClass.$metricClass")
 
@@ -46,12 +46,22 @@ object MetricsFactoryTemplate extends ((Seq[ModelMetricsSubstitutionContext]) =>
     }
   }
 
-  private def generatePatternMatchingCases(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
+  private def getEntityNames(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): Seq[String] = {
     metricSubstitutionContexts
       .map { metricSubstitutionContext =>
-        val metricsObjectName = metricSubstitutionContext.entityName
-        s"""        elif javaObject.getClass().getSimpleName() == "$metricsObjectName":
-           |            return $metricsObjectName(javaObject)""".stripMargin
+        if (metricSubstitutionContext.entityName.endsWith("Base")) {
+          metricSubstitutionContext.entityName.substring(0, metricSubstitutionContext.entityName.length - 4)
+        } else {
+          metricSubstitutionContext.entityName
+        }
+      }
+  }
+
+  private def generatePatternMatchingCases(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
+    getEntityNames(metricSubstitutionContexts)
+      .map { entityName =>
+        s"""        elif javaObject.getClass().getSimpleName() == "$entityName":
+           |            return $entityName(javaObject)""".stripMargin
       }
       .mkString("\n")
   }

api-generation/src/main/scala/ai/h2o/sparkling/api/generation/python/MetricsInitTemplate.scala

@@ -22,7 +22,13 @@ import ai.h2o.sparkling.api.generation.common.{EntitySubstitutionContext, ModelM
 object MetricsInitTemplate extends ((Seq[ModelMetricsSubstitutionContext]) => String) with PythonEntityTemplate {
 
   def apply(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
-    val metricClasses = metricSubstitutionContexts.map(_.entityName)
+    val metricClasses = metricSubstitutionContexts.map { metricSubstitutionContext =>
+      if (metricSubstitutionContext.entityName.endsWith("Base")) {
+        metricSubstitutionContext.entityName.substring(0, metricSubstitutionContext.entityName.length - 4)
+      } else {
+        metricSubstitutionContext.entityName
+      }
+    }
     val imports = metricClasses.map(metricClass => s"ai.h2o.sparkling.ml.metrics.$metricClass.$metricClass")
 
     val entitySubstitutionContext = EntitySubstitutionContext(null, null, null, imports)

api-generation/src/main/scala/ai/h2o/sparkling/api/generation/r/MetricsFactoryTemplate.scala

@@ -22,7 +22,7 @@ import ai.h2o.sparkling.api.generation.common.ModelMetricsSubstitutionContext
 object MetricsFactoryTemplate extends ((Seq[ModelMetricsSubstitutionContext]) => String) {
 
   def apply(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
-    val metricClasses = metricSubstitutionContexts.map(_.entityName)
+    val metricClasses = getEntityNames(metricSubstitutionContexts)
     val imports = metricClasses.map(metricClass => s"""source(file.path("R", "${metricClass}.R"))""").mkString("\n")
 
     s"""#
@@ -55,12 +55,23 @@ object MetricsFactoryTemplate extends ((Seq[ModelMetricsSubstitutionContext]) =>
        |""".stripMargin
   }
 
-  private def generateCases(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
+  private def getEntityNames(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): Seq[String] = {
     metricSubstitutionContexts
       .map { metricSubstitutionContext =>
-        val metricsObjectName = metricSubstitutionContext.entityName
-        s"""  } else if (invoke(invoke(javaObject, "getClass"), "getSimpleName") == "$metricsObjectName") {
-           |    rsparkling.$metricsObjectName(javaObject)""".stripMargin
+        if (metricSubstitutionContext.entityName.endsWith("Base")) {
+          metricSubstitutionContext.entityName.substring(0, metricSubstitutionContext.entityName.length - 4)
+        } else {
+          metricSubstitutionContext.entityName
+        }
+      }
+  }
+
+  private def generateCases(metricSubstitutionContexts: Seq[ModelMetricsSubstitutionContext]): String = {
+    val names = getEntityNames(metricSubstitutionContexts)
+    names
+      .map { entityName =>
+        s"""  } else if (invoke(invoke(javaObject, "getClass"), "getSimpleName") == "$entityName") {
+           |    rsparkling.$entityName(javaObject)""".stripMargin
       }
       .mkString("\n")
   }

core/src/test/scala/ai/h2o/sparkling/TestUtils.scala

@@ -24,7 +24,7 @@ import org.apache.spark.mllib
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
 import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
-import org.apache.spark.sql.functions.{lit, rand}
+import org.apache.spark.sql.functions.{lit, rand, col, abs}
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 import org.scalatest.Matchers
@@ -100,6 +100,50 @@ object TestUtils extends Matchers {
        """.stripMargin)
   }
 
+  def assertDataFramesAreEqual(
+      expected: DataFrame,
+      produced: DataFrame,
+      identityColumn: String,
+      tolerance: Double): Unit = {
+    val tolerances = expected.schema.fields
+      .filterNot(_.name == identityColumn)
+      .filter(_.dataType.isInstanceOf[NumericType])
+      .map(_.name -> tolerance)
+      .toMap
+    assertDataFramesAreEqual(expected, produced, identityColumn, tolerances)
+  }
+
+  def assertDataFramesAreEqual(
+      expected: DataFrame,
+      produced: DataFrame,
+      identityColumn: String,
+      tolerances: Map[String, Double] = Map.empty): Unit = {
+    expected.schema shouldEqual produced.schema
+    val intersection = expected.as("expected").join(produced.as("produced"), identityColumn)
+    intersection.count() shouldEqual expected.count()
+    intersection.count() shouldEqual produced.count()
+    val isEqualExpression = expected.columns.foldLeft(lit(true)) {
+      case (partialExpression, columnName) =>
+        val columnComparision = if (tolerances.contains(columnName)) {
+          val difference = abs(col(s"expected.$columnName") - col(s"produced.$columnName"))
+          difference <= lit(tolerances(columnName))
+        } else if (columnName == identityColumn) {
+          lit(true)
+        } else {
+          col(s"expected.$columnName") === col(s"produced.$columnName")
+        }
+        partialExpression && columnComparision
+    }
+    val withComparisonDF = intersection.withColumn("isEqual", isEqualExpression)
+    val differentRowsDF = withComparisonDF
+      .filter(col("isEqual") === lit(false))
+      .select(col(s"expected.$identityColumn") as "id")
+    val differentIds = differentRowsDF.collect().map(_.get(0))
+    assert(
+      differentIds.length == 0,
+      s"The rows of ids($identityColumn) [${differentIds.mkString(", ")}] are not equal.")
+  }
+
   def assertDatasetBasicProperties[T <: Product](
       ds: Dataset[T],
       df: H2OFrame,

doc/src/main/scala/ai/h2o/sparkling/doc/generation/Runner.scala

@@ -98,6 +98,7 @@ object Runner {
       }
     } else {
       val metricClasses = getParamClasses("ai.h2o.sparkling.ml.metrics")
+        .filter(_.getSimpleName.endsWith("Metrics"))
       writeResultToFile(MetricsTocTreeTemplate(metricClasses), "metrics", destinationDir)
       for (metricClass <- metricClasses) {
         val content = MetricsTemplate(metricClass)

extensions/src/main/resources/META-INF/services/water.TypeMapExtension

@@ -0,0 +1 @@
+hex.MetricsCalculationTypeExtensions

extensions/src/main/scala/hex/MetricsCalculationTypeExtensions.java

@@ -0,0 +1,57 @@
+package hex;
+
+import java.util.Arrays;
+import water.TypeMapExtension;
+import water.api.schemas3.*;
+
+public class MetricsCalculationTypeExtensions implements TypeMapExtension {
+  public static final String[] MODEL_BUILDER_CLASSES = {
+    ModelMetrics.MetricBuilder.class.getName(),
+    ModelMetricsSupervised.MetricBuilderSupervised.class.getName(),
+    ModelMetricsBinomial.MetricBuilderBinomial.class.getName(),
+    AUC2.AUCBuilder.class.getName(),
+    ModelMetricsRegression.MetricBuilderRegression.class.getName(),
+    Distribution.class.getName(),
+    GaussianDistribution.class.getName(),
+    BernoulliDistribution.class.getName(),
+    QuasibinomialDistribution.class.getName(),
+    ModifiedHuberDistribution.class.getName(),
+    MultinomialDistribution.class.getName(),
+    PoissonDistribution.class.getName(),
+    GammaDistribution.class.getName(),
+    TweedieDistribution.class.getName(),
+    HuberDistribution.class.getName(),
+    LaplaceDistribution.class.getName(),
+    QuantileDistribution.class.getName(),
+    CustomDistribution.class.getName(),
+    CustomDistributionWrapper.class.getName(),
+    LinkFunction.class.getName(),
+    IdentityFunction.class.getName(),
+    InverseFunction.class.getName(),
+    LogFunction.class.getName(),
+    LogitFunction.class.getName(),
+    OlogitFunction.class.getName(),
+    OloglogFunction.class.getName(),
+    OprobitFunction.class.getName(),
+    ModelMetricsMultinomial.MetricBuilderMultinomial.class.getName()
+  };
+
+  public static final String[] SCHEMA_CLASSES = {
+    ModelMetricsBaseV3.class.getName(),
+    ModelMetricsBinomialV3.class.getName(),
+    ModelMetricsMultinomialV3.class.getName(),
+    ModelMetricsRegressionV3.class.getName(),
+    ConfusionMatrixV3.class.getName(),
+    TwoDimTableV3.class.getName(),
+    TwoDimTableV3.ColumnSpecsBase.class.getName()
+  };
+
+  @Override
+  public String[] getBoostrapClasses() {
+    String[] result =
+        Arrays.copyOf(MODEL_BUILDER_CLASSES, MODEL_BUILDER_CLASSES.length + SCHEMA_CLASSES.length);
+    System.arraycopy(
+        SCHEMA_CLASSES, 0, result, MODEL_BUILDER_CLASSES.length, SCHEMA_CLASSES.length);
+    return result;
+  }
+}

ml/src/test/scala/ai/h2o/sparkling/ml/algos/BinomialPredictionTestSuite.scala

@@ -183,57 +183,4 @@ class BinomialPredictionTestSuite extends FunSuite with Matchers with SharedH2OT
     assert(schema == expectedSchema)
     assert(schema == expectedSchemaByTransform)
   }
-
-  private def assertMetrics[T](model: H2OMOJOModel): Unit = {
-    assertMetrics[T](model.getTrainingMetricsObject(), model.getTrainingMetrics())
-    assertMetrics[T](model.getValidationMetricsObject(), model.getValidationMetrics())
-    assert(model.getCrossValidationMetricsObject() == null)
-    assert(model.getCrossValidationMetrics() == Map())
-  }
-
-  private def assertMetrics[T](metricsObject: H2OMetrics, metrics: Map[String, Double]): Unit = {
-    metricsObject.isInstanceOf[T] should be(true)
-    MetricsAssertions.assertMetricsObjectAgainstMetricsMap(metricsObject, metrics)
-    val binomialObject = metricsObject.asInstanceOf[H2OBinomialMetrics]
-    binomialObject.getConfusionMatrix().count() > 0
-    binomialObject.getConfusionMatrix().columns.length > 0
-    binomialObject.getGainsLiftTable().count() > 0
-    binomialObject.getGainsLiftTable().columns.length > 0
-    binomialObject.getMaxCriteriaAndMetricScores().count() > 0
-    binomialObject.getMaxCriteriaAndMetricScores().columns.length > 0
-    binomialObject.getThresholdsAndMetricScores().count() > 0
-    binomialObject.getThresholdsAndMetricScores().columns.length > 0
-  }
-
-  test("test binomial metric objects") {
-    val algo = new H2OGBM()
-      .setSplitRatio(0.8)
-      .setSeed(1)
-      .setFeaturesCols("sepal_len", "sepal_wid")
-      .setColumnsToCategorical("class")
-      .setLabelCol("class")
-
-    val model = algo.fit(dataset)
-    assertMetrics[H2OBinomialMetrics](model)
-
-    model.write.overwrite().save("ml/build/gbm_binomial_model_metrics")
-    val loadedModel = H2OGBMMOJOModel.load("ml/build/gbm_binomial_model_metrics")
-    assertMetrics[H2OBinomialMetrics](loadedModel)
-  }
-
-  test("test binomial glm metric objects") {
-    val algo = new H2OGLM()
-      .setSplitRatio(0.8)
-      .setSeed(1)
-      .setFeaturesCols("sepal_len", "sepal_wid")
-      .setColumnsToCategorical("class")
-      .setLabelCol("class")
-
-    val model = algo.fit(dataset)
-    assertMetrics[H2OBinomialGLMMetrics](model)
-
-    model.write.overwrite().save("ml/build/glm_binomial_model_metrics")
-    val loadedModel = H2OGLMMOJOModel.load("ml/build/glm_binomial_model_metrics")
-    assertMetrics[H2OBinomialGLMMetrics](loadedModel)
-  }
 }

ml/src/test/scala/ai/h2o/sparkling/ml/algos/RegressionPredictionTestSuite.scala

@@ -153,38 +153,4 @@ class RegressionPredictionTestSuite extends FunSuite with Matchers with SharedH2
     metricsObject.isInstanceOf[T] should be(true)
     MetricsAssertions.assertMetricsObjectAgainstMetricsMap(metricsObject, metrics)
   }
-
-  test("test regression metric objects") {
-    val algo = new algos.H2OGBM()
-      .setSplitRatio(0.8)
-      .setSeed(1)
-      .setWithContributions(true)
-      .setWithLeafNodeAssignments(true)
-      .setWithStageResults(true)
-      .setFeaturesCols("CAPSULE", "RACE", "DPROS", "DCAPS", "PSA", "VOL", "GLEASON")
-      .setLabelCol("AGE")
-    val model = algo.fit(dataset)
-    assertMetrics[H2ORegressionMetrics](model)
-
-    model.write.overwrite().save("ml/build/gbm_regression_model_metrics")
-    val loadedModel = H2OGBMMOJOModel.load("ml/build/gbm_regression_model_metrics")
-    assertMetrics[H2ORegressionMetrics](loadedModel)
-  }
-
-  test("test regression glm metric objects") {
-    val algo = new algos.H2OGLM()
-      .setSplitRatio(0.8)
-      .setSeed(1)
-      .setWithContributions(true)
-      .setWithLeafNodeAssignments(true)
-      .setWithStageResults(true)
-      .setFeaturesCols("CAPSULE", "RACE", "DPROS", "DCAPS", "PSA", "VOL", "GLEASON")
-      .setLabelCol("AGE")
-    val model = algo.fit(dataset)
-    assertMetrics[H2ORegressionGLMMetrics](model)
-
-    model.write.overwrite().save("ml/build/glm_regression_model_metrics")
-    val loadedModel = H2OGLMMOJOModel.load("ml/build/glm_regression_model_metrics")
-    assertMetrics[H2ORegressionGLMMetrics](loadedModel)
-  }
 }