v0.2.52..v0.2.53 changeset ConflateCmd.cpp

diff --git a/hoot-core/src/main/cpp/hoot/core/cmd/ConflateCmd.cpp b/hoot-core/src/main/cpp/hoot/core/cmd/ConflateCmd.cpp
index 1b0f283..6db7093 100644
--- a/hoot-core/src/main/cpp/hoot/core/cmd/ConflateCmd.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/cmd/ConflateCmd.cpp
@@ -22,7 +22,7 @@
  * This will properly maintain the copyright information. DigitalGlobe
  * copyrights will be updated automatically.
  *
- * @copyright Copyright (C) 2015, 2017, 2018, 2019 DigitalGlobe (http://www.digitalglobe.com/)
+ * @copyright Copyright (C) 2015, 2017, 2018, 2019, 2020 DigitalGlobe (http://www.digitalglobe.com/)
  */
 
 #include "ConflateCmd.h"
@@ -64,7 +64,10 @@
 // Tgs
 #include <tgs/System/Timer.h>
 
+// Qt
 #include <QFileInfo>
+#include <QDir>
+#include <QElapsedTimer>
 
 using namespace std;
 using namespace Tgs;
@@ -162,6 +165,14 @@ int ConflateCmd::runSimple(QStringList& args)
   const QString input2 = args[1];
   QString output = args[2];
 
+  QFileInfo outputInfo(output);
+  LOG_VARD(outputInfo.dir().absolutePath());
+  const bool outputDirSuccess = QDir().mkpath(outputInfo.dir().absolutePath());
+  if (!outputDirSuccess)
+  {
+    throw IllegalArgumentException("Unable to create output path for: " + output);
+  }
+
   QString osmApiDbUrl;
   if (output.endsWith(".osc.sql"))
   {
@@ -204,10 +215,16 @@ int ConflateCmd::runSimple(QStringList& args)
   LOG_VART(bytesRead);
   QList<QList<SingleStat>> allStats;
 
-  _updateConfigOptionsForAttributeConflation();
-  if (isDiffConflate)
+  // The highway.merge.tags.only option only gets used with Attribute Conflation for now, so we'll
+  // use it as the sole identifier for it. If that ever changes, then we'll need a different way
+  // to recognize when AC is occurring.
+  if (ConfigOptions().getHighwayMergeTagsOnly())
+  {
+    _updateConfigOptionsForAttributeConflation();
+  }
+  if (isDiffConflate || ConfigOptions().getHighwayMergeTagsOnly())
   {
-    _updateConfigOptionsForDifferentialConflation();
+    _disableRoundaboutRemoval();
   }
 
   // The number of steps here must be updated as you add/remove job steps in the logic.
@@ -220,6 +237,7 @@ int ConflateCmd::runSimple(QStringList& args)
   {
     _numTotalTasks++;
   }
+
   // Only add one task for each set of conflate ops, since NamedOp will create its own task step for
   // each op internally.
   if (ConfigOptions().getConflatePreOps().size() > 0)
@@ -290,7 +308,7 @@ int ConflateCmd::runSimple(QStringList& args)
       map, input2, ConfigOptions().getConflateUseDataSourceIds2(), Status::Unknown2);
     currentTask++;
   }
-  LOG_INFO("Conflating map with " << StringUtils::formatLargeNumber(map->size()) << " elements...");
+  LOG_STATUS("Conflating map with " << StringUtils::formatLargeNumber(map->size()) << " elements...");
 
   double inputBytes = IoSingleStat(IoSingleStat::RChar).value - bytesRead;
   LOG_VART(inputBytes);
@@ -329,6 +347,8 @@ int ConflateCmd::runSimple(QStringList& args)
   if (ConfigOptions().getConflatePreOps().size() > 0)
   {
     // apply any user specified pre-conflate operations
+    QElapsedTimer timer;
+    timer.start();
     NamedOp preOps(ConfigOptions().getConflatePreOps());
     preOps.setProgress(
       Progress(
@@ -338,6 +358,9 @@ int ConflateCmd::runSimple(QStringList& args)
     stats.append(SingleStat("Apply Pre-Conflate Ops Time (sec)", t.getElapsedAndRestart()));
     OsmMapWriterFactory::writeDebugMap(map, "after-pre-ops");
     currentTask++;
+    LOG_STATUS(
+      "Conflate pre-operations ran in " + StringUtils::millisecondsToDhms(timer.elapsed()) <<
+      " total.");
   }
 
   OsmMapPtr result = map;
@@ -371,6 +394,8 @@ int ConflateCmd::runSimple(QStringList& args)
   if (ConfigOptions().getConflatePostOps().size() > 0)
   {
     // apply any user specified post-conflate operations
+    QElapsedTimer timer;
+    timer.start();
     NamedOp postOps(ConfigOptions().getConflatePostOps());
     postOps.setProgress(
       Progress(
@@ -380,6 +405,9 @@ int ConflateCmd::runSimple(QStringList& args)
     stats.append(SingleStat("Apply Post-Conflate Ops Time (sec)", t.getElapsedAndRestart()));
     OsmMapWriterFactory::writeDebugMap(result, "after-post-ops");
     currentTask++;
+    LOG_STATUS(
+      "Conflate post-operations ran in " + StringUtils::millisecondsToDhms(timer.elapsed()) <<
+      " total.");
   }
 
   // doing this after the conflate post ops run, since some invalid reviews are removed by them
@@ -472,16 +500,15 @@ int ConflateCmd::runSimple(QStringList& args)
 
   if (displayStats)
   {
+    allStats.append(stats);
     if (outputStatsFile.isEmpty())
     {
-      allStats.append(stats);
       QString statsMsg = MapStatsWriter().statsToString(allStats, "\t");
       cout << "stats = (stat) OR (input map 1 stat) (input map 2 stat) (output map stat)\n" <<
               statsMsg << endl;
     }
     else
     {
-      allStats.append(stats);
       MapStatsWriter().writeStatsToJson(allStats, outputStatsFile);
       cout << "stats = (stat) OR (input map 1 stat) (input map 2 stat) (output map stat) in file: " <<
               outputStatsFile << endl;
@@ -509,12 +536,23 @@ float ConflateCmd::_getJobPercentComplete(const int currentTaskNum) const
   return (float)currentTaskNum / (float)_numTotalTasks;
 }
 
-void ConflateCmd::_updateConfigOptionsForDifferentialConflation()
+void ConflateCmd::_disableRoundaboutRemoval()
 {
+  // This applies to both Attribute and Differential Conflation.
+
   // Since Differential throws out all matches, there's no way we can have a bad merge between
   // ref/secondary roundabouts. Therefore, no need to replace/remove them. If there's a match, we'll
-  // end with no secondary roundabout in the diff output and only the ref roundabout when the diff
-  // is applied back to the ref.
+  // end up with no secondary roundabout in the diff output and only the ref roundabout when the
+  // diff is applied back to the ref.
+
+  // Our roundabout handling used with other types of conflation makes no sense for Attribute
+  // Conflation. For the other types of conflation we remove roundabouts b/c hoot isn't very good
+  // at merging them together, so we keep the ref roundabouts and skip conflating them altogether.
+  // Since we always keep the ref geometry in AC, there's no opportunity for bad merging and,
+  // thus, no need to remove them in the first place. So, we'll override that behavior here.
+
+  // If the work for #3442 is done, then we could handle this removal in AttributeConflation.conf
+  // add DifferentialConflation.conf instead of doing it here.
 
   QStringList preConflateOps = ConfigOptions().getConflatePreOps();
   const QString removeRoundaboutsClassName = QString::fromStdString(RemoveRoundabouts::className());
@@ -539,25 +577,19 @@ void ConflateCmd::_updateConfigOptionsForAttributeConflation()
   // These are some custom adjustments to config opts that must be done for Attribute Conflation.
   // There may be a way to eliminate these by adding more custom behavior to the UI.
 
-  // This option only gets used with Attribute Conflation for now, so we'll use it as the sole
-  // identifier for it.  This could change in the future, but hopefully if that happens, by then
-  // we have a better solution for changing these opts in place.
-  if (ConfigOptions().getHighwayMergeTagsOnly())
-  {
-    const QString reviewRelationCritName =
-      QString::fromStdString(ReviewRelationCriterion::className());
+  const QString reviewRelationCritName =
+    QString::fromStdString(ReviewRelationCriterion::className());
 
-    // This swaps the logic that removes all reviews with the logic that removes them based on score
-    // thresholding.
-    if (ConfigOptions().getAttributeConflationAllowReviewsByScore())
-    {
-      QStringList removeElementsCriteria =
-        conf().get(ConfigOptions::getRemoveElementsVisitorElementCriteriaKey()).toStringList();
-      removeElementsCriteria.replaceInStrings(
-        reviewRelationCritName, QString::fromStdString(ReviewScoreCriterion::className()));
-      conf().set(
-        ConfigOptions::getRemoveElementsVisitorElementCriteriaKey(), removeElementsCriteria);
-    }
+  // This swaps the logic that removes all reviews with the logic that removes them based on score
+  // thresholding.
+  if (ConfigOptions().getAttributeConflationAllowReviewsByScore())
+  {
+    QStringList removeElementsCriteria =
+      conf().get(ConfigOptions::getRemoveElementsVisitorElementCriteriaKey()).toStringList();
+    removeElementsCriteria.replaceInStrings(
+      reviewRelationCritName, QString::fromStdString(ReviewScoreCriterion::className()));
+    conf().set(
+      ConfigOptions::getRemoveElementsVisitorElementCriteriaKey(), removeElementsCriteria);
   }
 }