v0.2.47..v0.2.48 changeset OsmApiWriter.cpp

diff --git a/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp b/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
index 4c41c5c..209d530 100644
--- a/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/io/OsmApiWriter.cpp
@@ -48,6 +48,13 @@ using namespace Tgs;
 namespace hoot
 {
 
+const char* OsmApiWriter::API_PATH_CAPABILITIES = "/api/capabilities/";
+const char* OsmApiWriter::API_PATH_PERMISSIONS = "/api/0.6/permissions/";
+const char* OsmApiWriter::API_PATH_CREATE_CHANGESET = "/api/0.6/changeset/create/";
+const char* OsmApiWriter::API_PATH_CLOSE_CHANGESET = "/api/0.6/changeset/%1/close/";
+const char* OsmApiWriter::API_PATH_UPLOAD_CHANGESET = "/api/0.6/changeset/%1/upload/";
+const char* OsmApiWriter::API_PATH_GET_ELEMENT = "/api/0.6/%1/%2/";
+
 OsmApiWriter::OsmApiWriter(const QUrl &url, const QString &changeset)
   : _description(ConfigOptions().getChangesetDescription()),
     _maxWriters(ConfigOptions().getChangesetApidbWritersMax()),
@@ -90,6 +97,10 @@ bool OsmApiWriter::apply()
     return false;
   }
   _stats.append(SingleStat("API Capabilites Query Time (sec)", timer.getElapsedAndRestart()));
+  //  Limit the max changeset size to the max from the API, testing found that smaller (1k to 2k elements)
+  //  pushed across multiple processing threads out performed larger (50k element) datasets
+  if (_maxChangesetSize > _capabilities.getChangesets())
+    _maxChangesetSize = _capabilities.getChangesets();
   //  Setup the network request object with OAuth or with username/password authentication
   request = createNetworkRequest(true);
   //  Validate API permissions
@@ -254,9 +265,20 @@ void OsmApiWriter::_changesetThreadFunc()
         //  Split the changeset on conflict errors
         switch (info->status)
         {
+        case 409:   //  Conflict, check for version conflicts and fix, or split and continue
+          {
+            if (_fixConflict(request, workInfo, info->response))
+            {
+              _workQueueMutex.lock();
+              _workQueue.push(workInfo);
+              _workQueueMutex.unlock();
+              //  Loop back around to work on the next changeset
+              continue;
+            }
+            //  Fall through here to split the changeset and retry
+          }
         case 400:   //  Placeholder ID is missing or not unique
         case 404:   //  Diff contains elements where the given ID could not be found
-        case 409:   //  Conflict, Split the changeset, push both back on the queue
         case 412:   //  Precondition Failed, Relation with id cannot be saved due to other member
           {
             _changesetMutex.lock();
@@ -271,10 +293,10 @@ void OsmApiWriter::_changesetThreadFunc()
             }
             else
             {
-              if (!workInfo->getChangesetIssuesResolved())
+              if (!workInfo->getAttemptedResolveChangesetIssues())
               {
-                //  Set the issues resolved flag
-                workInfo->setChangesetIssuesResolved(true);
+                //  Set the attempt issues resolved flag
+                workInfo->setAttemptedResolveChangesetIssues(true);
                 //  Try to automatically resolve certain issues, like out of date version
                 if (_resolveIssues(request, workInfo))
                 {
@@ -291,17 +313,20 @@ void OsmApiWriter::_changesetThreadFunc()
             }
           }
           break;
-        case 405:   //  This shouldn't ever happen, push back on the queue
-          _workQueueMutex.lock();
-          _workQueue.push(workInfo);
-          _workQueueMutex.unlock();
-          break;
         default:
           //  This is a big problem, report it and try again
           LOG_ERROR("Changeset upload responded with HTTP status response: " << request->getHttpStatus());
-          _workQueueMutex.lock();
-          _workQueue.push(workInfo);
-          _workQueueMutex.unlock();
+        case 405:
+          //  This shouldn't ever happen, push back on the queue, only process a certain amount of times
+          workInfo->retry();
+          if (workInfo->canRetry())
+          {
+            _workQueueMutex.lock();
+            _workQueue.push(workInfo);
+            _workQueueMutex.unlock();
+          }
+          else
+            _changeset.updateFailedChangeset(workInfo, true);
           break;
         }
       }
@@ -491,7 +516,7 @@ void OsmApiWriter::_closeChangeset(HootNetworkRequestPtr request, long id)
   try
   {
     QUrl changeset = _url;
-    changeset.setPath(API_PATH_CLOSE_CHANGESET.arg(id));
+    changeset.setPath(QString(API_PATH_CLOSE_CHANGESET).arg(id));
     request->networkRequest(changeset, QNetworkAccessManager::Operation::PutOperation);
     QString responseXml = QString::fromUtf8(request->getResponseContent().data());
     switch (request->getHttpStatus())
@@ -547,7 +572,7 @@ OsmApiWriter::OsmApiFailureInfoPtr OsmApiWriter::_uploadChangeset(HootNetworkReq
   try
   {
     QUrl change = _url;
-    change.setPath(API_PATH_UPLOAD_CHANGESET.arg(id));
+    change.setPath(QString(API_PATH_UPLOAD_CHANGESET).arg(id));
 
     QMap<QNetworkRequest::KnownHeaders, QVariant> headers;
     headers[QNetworkRequest::ContentTypeHeader] = "text/xml";
@@ -586,6 +611,36 @@ OsmApiWriter::OsmApiFailureInfoPtr OsmApiWriter::_uploadChangeset(HootNetworkReq
   return info;
 }
 
+bool OsmApiWriter::_fixConflict(HootNetworkRequestPtr request, ChangesetInfoPtr changeset, const QString& conflictExplanation)
+{
+  bool success = false;
+  long element_id = 0;
+  ElementType::Type element_type = ElementType::Unknown;
+  long version_old = 0;
+  long version_new = 0;
+  if (_changeset.matchesChangesetConflictVersionMismatchFailure(conflictExplanation, element_id, element_type, version_old, version_new))
+  {
+    for (int changesetType = XmlChangeset::TypeModify; changesetType < XmlChangeset::TypeMax; ++changesetType)
+    {
+      ChangesetInfo::iterator element = changeset->begin((ElementType::Type)element_type, (XmlChangeset::ChangesetType)changesetType);
+      if (element != changeset->end((ElementType::Type)element_type, (XmlChangeset::ChangesetType)changesetType))
+      {
+        QString update = "";
+        //  Get the element from the OSM API
+        if (element_type == ElementType::Node)
+          update = _getNode(request, element_id);
+        else if (element_type == ElementType::Way)
+          update = _getWay(request, element_id);
+        else if (element_type == ElementType::Relation)
+          update = _getRelation(request, element_id);
+        //  Fix the changeset with the node/way/relation from the OSM API
+        success |= _changeset.fixChangeset(update);
+      }
+    }
+  }
+  return success;
+}
+
 bool OsmApiWriter::_resolveIssues(HootNetworkRequestPtr request, ChangesetInfoPtr changeset)
 {
   bool success = false;