v0.2.53..v0.2.54 changeset OsmApiChangeset.cpp
Garret Voltz edited this page Mar 31, 2020
·
1 revision
diff --git a/hoot-core/src/main/cpp/hoot/core/io/OsmApiChangeset.cpp b/hoot-core/src/main/cpp/hoot/core/io/OsmApiChangeset.cpp
index e425d27..2c9087d 100644
--- a/hoot-core/src/main/cpp/hoot/core/io/OsmApiChangeset.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/io/OsmApiChangeset.cpp
@@ -32,6 +32,7 @@
#include <hoot/core/util/ConfigOptions.h>
#include <hoot/core/util/FileUtils.h>
#include <hoot/core/util/Log.h>
+#include <hoot/core/util/StringUtils.h>
// Standard
#include <algorithm>
@@ -50,13 +51,27 @@ XmlChangeset::XmlChangeset()
_ways(ChangesetType::TypeMax),
_relations(ChangesetType::TypeMax),
_maxPushSize(ConfigOptions().getChangesetApidbSizeMax()),
+ _maxChangesetSize(ConfigOptions().getChangesetMaxSize()),
_sentCount(0),
_processedCount(0),
_failedCount(0)
{
}
-XmlChangeset::XmlChangeset(const QList<QString> &changesets)
+XmlChangeset::XmlChangeset(const QString& changeset)
+ : _nodes(ChangesetType::TypeMax),
+ _ways(ChangesetType::TypeMax),
+ _relations(ChangesetType::TypeMax),
+ _maxPushSize(ConfigOptions().getChangesetApidbSizeMax()),
+ _maxChangesetSize(ConfigOptions().getChangesetMaxSize()),
+ _sentCount(0),
+ _processedCount(0),
+ _failedCount(0)
+{
+ loadChangeset(changeset);
+}
+
+XmlChangeset::XmlChangeset(const QList<QString>& changesets)
: _nodes(ChangesetType::TypeMax),
_ways(ChangesetType::TypeMax),
_relations(ChangesetType::TypeMax),
@@ -69,10 +84,16 @@ XmlChangeset::XmlChangeset(const QList<QString> &changesets)
loadChangeset(*it);
}
-void XmlChangeset::loadChangeset(const QString &changeset)
+void XmlChangeset::loadChangeset(const QString& changeset)
{
- if (QFile::exists(changeset))
- loadChangesetFile(changeset);
+ QFileInfo fi(changeset);
+ if (fi.exists())
+ {
+ if (fi.isDir())
+ loadChangesetDirectory(changeset);
+ else
+ loadChangesetFile(changeset);
+ }
else
loadChangesetXml(changeset);
}
@@ -95,7 +116,7 @@ void XmlChangeset::loadChangesetFile(const QString& path)
loadOsmAsChangeset(reader);
}
-void XmlChangeset::loadChangesetXml(const QString &changesetXml)
+void XmlChangeset::loadChangesetXml(const QString& changesetXml)
{
// Load the XML directly into the reader
QXmlStreamReader reader(changesetXml);
@@ -103,7 +124,49 @@ void XmlChangeset::loadChangesetXml(const QString &changesetXml)
loadChangeset(reader);
}
-void XmlChangeset::loadChangeset(QXmlStreamReader &reader)
+void XmlChangeset::loadChangesetDirectory(const QString& changesetDirectory)
+{
+ // Iterate all of the files in the directory, loading them one by one
+ QDir dir(changesetDirectory);
+ dir.setFilter(QDir::Files | QDir::NoDotAndDotDot);
+ dir.setSorting(QDir::Name | QDir::IgnoreCase);
+ QStringList filters;
+ filters << "*.osc";
+ dir.setNameFilters(filters);
+
+ QFileInfoList files = dir.entryInfoList();
+ for (int i = 0; i < files.size(); ++i)
+ {
+ QFileInfo fileInfo = files.at(i);
+ QString filepath = fileInfo.absoluteFilePath();
+ // Check if this file is in the request/response debug output format (including the error file)
+ if (filepath.contains("-Request--") && filepath.endsWith("000.osc", Qt::CaseInsensitive))
+ {
+ // Only load the requests that were successful, i.e. response 200
+ QString response = filepath;
+ response.replace("-Request--", "-Response-").replace("000.osc", "200.osc");
+ if (QFile::exists(response))
+ loadChangesetFile(filepath);
+ }
+ else if (filepath.contains("-Response-") && filepath.endsWith("200.osc", Qt::CaseInsensitive))
+ {
+ // Read in the update for new IDs
+ updateChangeset(FileUtils::readFully(filepath));
+ }
+ else if (filepath.endsWith("-error.osc", Qt::CaseInsensitive))
+ {
+ // The changeset error file
+ loadChangesetFile(filepath);
+ }
+ else
+ {
+ // Output the filename to the log
+ LOG_DEBUG("Skipping file in folder: " << filepath);
+ }
+ }
+}
+
+void XmlChangeset::loadChangeset(QXmlStreamReader& reader)
{
// Make sure that the XML provided starts with the <diffResult> tag
QXmlStreamReader::TokenType type = reader.readNext();
@@ -314,7 +377,18 @@ void XmlChangeset::fixMalformedInput()
writeErrorFile();
}
-void XmlChangeset::updateChangeset(const QString &changes)
+QString XmlChangeset::getString(ChangesetType type)
+{
+ switch (type)
+ {
+ case ChangesetType::TypeCreate: return "<create>";
+ case ChangesetType::TypeModify: return "<modify>";
+ case ChangesetType::TypeDelete: return "<delete>";
+ default: return "";
+ }
+}
+
+void XmlChangeset::updateChangeset(const QString& changes)
{
/* <diffResult generator="OpenStreetMap Server" version="0.6">
* <node|way|relation old_id="#" new_id="#" new_version="#"/>
@@ -361,41 +435,10 @@ void XmlChangeset::updateChangeset(const QString &changes)
updateElement(_ways, old_id, new_id, version);
else if (name == "relation")
updateElement(_relations, old_id, new_id, version);
- }
- }
-}
-
-void XmlChangeset::updateChangeset(const ChangesetInfoPtr& changeset_info)
-{
- // Iterate the three changeset type arrays looking for elements to mark
- for (int current_type = ChangesetType::TypeCreate; current_type != ChangesetType::TypeMax; ++current_type)
- {
- // Set the relation's status to failed
- for (ChangesetInfo::iterator it = changeset_info->begin(ElementType::Relation, (ChangesetType)current_type);
- it != changeset_info->end(ElementType::Relation, (ChangesetType)current_type); ++it)
- {
- // Finalize the relation
- _allRelations[*it]->setStatus(ChangesetElement::ElementStatus::Finalized);
- // Update the processed count
- _processedCount++;
- }
- // Set the way's status to failed
- for (ChangesetInfo::iterator it = changeset_info->begin(ElementType::Way, (ChangesetType)current_type);
- it != changeset_info->end(ElementType::Way, (ChangesetType)current_type); ++it)
- {
- // Finalize the way
- _allWays[*it]->setStatus(ChangesetElement::ElementStatus::Finalized);
- // Update the processed count
- _processedCount++;
- }
- // Set the node's status to failed
- for (ChangesetInfo::iterator it = changeset_info->begin(ElementType::Node, (ChangesetType)current_type);
- it != changeset_info->end(ElementType::Node, (ChangesetType)current_type); ++it)
- {
- // Finalize the node
- _allNodes[*it]->setStatus(ChangesetElement::ElementStatus::Finalized);
- // Update the processed count
- _processedCount++;
+ if (old_id == 0)
+ {
+ LOG_ERROR("Element cannot be updated. No ID given.");
+ }
}
}
}
@@ -491,6 +534,7 @@ bool XmlChangeset::addNode(ChangesetInfoPtr& changeset, ChangesetType type, Chan
// Only add the nodes that are "sendable"
if (canSend(node))
{
+ // No need to getObjectCount() like addWay and addRelation because it is either 0 or 1
// Add create nodes if the ID map's ID is negative, modify IDs don't matter
if ((type == ChangesetType::TypeCreate && _idMap.getId(ElementType::Node, node->id()) < 0) ||
type != ChangesetType::TypeDelete)
@@ -572,8 +616,14 @@ bool XmlChangeset::addWays(ChangesetInfoPtr& changeset, ChangesetType type)
bool XmlChangeset::addWay(ChangesetInfoPtr& changeset, ChangesetType type, ChangesetWay* way)
{
+ // Check if the way is able to be sent
if (canSend(way))
{
+ // The number of elements in this way (fully "hydrated") cannot exceed the max size in a changeset
+ ElementCountSet elements(ElementType::Max);
+ size_t count = getObjectCount(way, elements);
+ if (count + changeset->size() > (size_t)_maxChangesetSize)
+ return false;
bool sendable = true;
// Only creates/modifies require pre-processing
if (type != ChangesetType::TypeDelete)
@@ -695,7 +745,8 @@ bool XmlChangeset::canMoveWay(ChangesetInfoPtr& source, ChangesetInfoPtr& /*dest
if (type == ChangesetType::TypeDelete)
return source->size() == 1;
// Get the count of nodes and way that make up this "change"
- size_t count = getObjectCount(source, way);
+ ElementCountSet elements(ElementType::Max);
+ size_t count = getObjectCount(source, way, elements);
// Compare that count to the size left in the source changeset
return source->size() != count;
}
@@ -716,8 +767,14 @@ bool XmlChangeset::addRelations(ChangesetInfoPtr& changeset, ChangesetType type)
bool XmlChangeset::addRelation(ChangesetInfoPtr& changeset, ChangesetType type, ChangesetRelation* relation)
{
+ // Check if the relation is able to be sent
if (canSend(relation))
{
+ // The number of elements in this relation (fully "hydrated") cannot exceed the max size in a changeset
+ ElementCountSet elements(ElementType::Max);
+ size_t count = getObjectCount(relation, elements);
+ if (count + changeset->size() > (size_t)_maxChangesetSize)
+ return false;
bool sendable = true;
// Deletes require no pre-processing
if (type != ChangesetType::TypeDelete)
@@ -895,43 +952,89 @@ bool XmlChangeset::canMoveRelation(ChangesetInfoPtr& source, ChangesetInfoPtr& /
if (type == ChangesetType::TypeDelete)
return source->size() == 1;
// Get the count of nodes, ways, and relations that make up this "change"
- size_t count = getObjectCount(source, relation);
+ ElementCountSet elements(ElementType::Max);
+ size_t count = getObjectCount(source, relation, elements);
// Compare that count to the size left in the source changeset
return source->size() != count;
}
-size_t XmlChangeset::getObjectCount(ChangesetInfoPtr& /*changeset*/, ChangesetNode* node)
+size_t XmlChangeset::getObjectCount(ChangesetNode* node, ElementCountSet& elements)
{
+ ChangesetInfoPtr empty;
+ return getObjectCount(empty, node, elements);
+}
+
+size_t XmlChangeset::getObjectCount(ChangesetWay* way, ElementCountSet& elements)
+{
+ ChangesetInfoPtr empty;
+ return getObjectCount(empty, way, elements);
+}
+
+size_t XmlChangeset::getObjectCount(ChangesetRelation* relation, ElementCountSet& elements)
+{
+ ChangesetInfoPtr empty;
+ return getObjectCount(empty, relation, elements);
+}
+
+size_t XmlChangeset::getObjectCount(ChangesetInfoPtr& /*changeset*/, ChangesetNode* node, ElementCountSet& elements)
+{
+ // Cannot count NULL nodes
if (node == NULL)
return 0;
+ // Do not recount nodes already counted
+ if (elements[ElementType::Node].find(node->id()) != elements[ElementType::Node].end())
+ return 0;
+ // Record the node ID in the node elements set
+ elements[ElementType::Node].insert(node->id());
// Nodes count as one object
return 1;
}
-size_t XmlChangeset::getObjectCount(ChangesetInfoPtr& changeset, ChangesetWay* way)
+size_t XmlChangeset::getObjectCount(ChangesetInfoPtr& changeset, ChangesetWay* way, ElementCountSet& elements)
{
if (way == NULL)
return 0;
// Get the count of nodes and way that make up this "change"
- size_t count = 1;
+ size_t count = 0;
+ // Increment the count if it isn't already found in the element set
+ if (elements[ElementType::Way].find(way->id()) == elements[ElementType::Way].end())
+ {
+ elements[ElementType::Way].insert(way->id());
+ ++count;
+ }
+ // Iterate all of the nodes
for (int i = 0; i < way->getNodeCount(); ++i)
{
long id = way->getNode(i);
+ // Look for the node in the list of nodes
for (int current_type = ChangesetType::TypeCreate; current_type != ChangesetType::TypeMax; ++current_type)
{
- if (changeset->contains(ElementType::Node, (ChangesetType)current_type, id))
- ++count;
+ // Do not recount this node
+ if (!changeset || changeset->contains(ElementType::Node, (ChangesetType)current_type, id))
+ {
+ ChangesetNode* node = NULL;
+ if (_allNodes.find(id) != _allNodes.end())
+ node = dynamic_cast<ChangesetNode*>(_allNodes[id].get());
+ count += getObjectCount(changeset, node, elements);
+ }
}
}
return count;
}
-size_t XmlChangeset::getObjectCount(ChangesetInfoPtr& changeset, ChangesetRelation* relation)
+size_t XmlChangeset::getObjectCount(ChangesetInfoPtr& changeset, ChangesetRelation* relation, ElementCountSet& elements)
{
if (relation == NULL)
return 0;
// Get the count of nodes, ways, and relations that make up this "change"
- size_t count = 1;
+ size_t count = 0;
+ // Increment the count if the relation isn't already found in the element set
+ if (elements[ElementType::Relation].find(relation->id()) == elements[ElementType::Relation].end())
+ {
+ elements[ElementType::Relation].insert(relation->id());
+ ++count;
+ }
+ // Iterate all of the relation elements
for (int i = 0; i < relation->getMemberCount(); ++i)
{
ChangesetRelationMember& member = relation->getMember(i);
@@ -941,16 +1044,26 @@ size_t XmlChangeset::getObjectCount(ChangesetInfoPtr& changeset, ChangesetRelati
{
for (int current_type = ChangesetType::TypeCreate; current_type != ChangesetType::TypeMax; ++current_type)
{
- if (changeset->contains(ElementType::Node, (ChangesetType)current_type, id))
- count += getObjectCount(changeset, dynamic_cast<ChangesetNode*>(_allNodes[id].get()));
+ if (!changeset || changeset->contains(ElementType::Node, (ChangesetType)current_type, id))
+ {
+ ChangesetNode* node = NULL;
+ if (_allNodes.find(id) != _allNodes.end())
+ node = dynamic_cast<ChangesetNode*>(_allNodes[id].get());
+ count += getObjectCount(changeset, node, elements);
+ }
}
}
else if (member.isWay())
{
for (int current_type = ChangesetType::TypeCreate; current_type != ChangesetType::TypeMax; ++current_type)
{
- if (changeset->contains(ElementType::Way, (ChangesetType)current_type, id))
- count += getObjectCount(changeset, dynamic_cast<ChangesetWay*>(_allWays[id].get()));
+ if (!changeset || changeset->contains(ElementType::Way, (ChangesetType)current_type, id))
+ {
+ ChangesetWay* way = NULL;
+ if (_allWays.find(id) != _allWays.end())
+ way = dynamic_cast<ChangesetWay*>(_allWays[id].get());
+ count += getObjectCount(changeset, way, elements);
+ }
}
}
else if (member.isRelation())
@@ -960,8 +1073,13 @@ size_t XmlChangeset::getObjectCount(ChangesetInfoPtr& changeset, ChangesetRelati
{
for (int current_type = ChangesetType::TypeCreate; current_type != ChangesetType::TypeMax; ++current_type)
{
- if (changeset->contains(ElementType::Relation, (ChangesetType)current_type, id))
- count += getObjectCount(changeset, dynamic_cast<ChangesetRelation*>(_allRelations[id].get()));
+ if (!changeset || changeset->contains(ElementType::Relation, (ChangesetType)current_type, id))
+ {
+ ChangesetRelation* relation = NULL;
+ if (_allRelations.find(id) != _allRelations.end())
+ relation = dynamic_cast<ChangesetRelation*>(_allRelations[id].get());
+ count += getObjectCount(changeset, relation, elements);
+ }
}
}
}
@@ -1576,10 +1694,9 @@ QString XmlChangeset::getChangeset(ChangesetInfoPtr changeset, long changeset_id
category = "delete";
if (changeset->size(ElementType::Node, type) > 0 || changeset->size(ElementType::Way, type) > 0 || changeset->size(ElementType::Relation, type) > 0)
{
- ts << "\t<" << category;
+ ts << "\t<" << category << ">\n";
if (type != ChangesetType::TypeDelete)
{
- ts << ">\n";
// Nodes go first in each category
writeNodes(changeset, ts, type, changeset_id);
// Followed by ways
@@ -1589,7 +1706,6 @@ QString XmlChangeset::getChangeset(ChangesetInfoPtr changeset, long changeset_id
}
else
{
- ts << " if-unused=\"true\">\n";
// Relations first for deletes
writeRelations(changeset, ts, type, changeset_id);
// Followed by ways
@@ -1631,6 +1747,10 @@ void XmlChangeset::updateElement(ChangesetTypeMap& map, long old_id, long new_id
element->setVersion(version);
_processedCount++;
}
+ else
+ {
+ LOG_ERROR("Element cannot be updated. ID " << old_id);
+ }
}
bool XmlChangeset::fixElement(ChangesetTypeMap& map, long id, long version, QMap<QString, QString> tags)
@@ -1844,8 +1964,149 @@ bool XmlChangeset::calculateRemainingChangeset(ChangesetInfoPtr &changeset)
}
// Clear the send buffer
_sendBuffer.clear();
+ bool empty_changeset = changeset->size() == 0;
+ // Output the remaining changeset to a file to inform the user where the issues lie
+ if (!empty_changeset && !_errorPathname.isEmpty())
+ {
+ // Replace error with remaining in the error pathname
+ QString pathname = _errorPathname;
+ pathname.replace("error", "remaining");
+ // Write the file with changeset ID of zero
+ FileUtils::writeFully(pathname, this->getChangesetString(changeset, 0));
+ }
// Return true if there is anything in the changeset
- return changeset->size() > 0;
+ return !empty_changeset;
+}
+
+bool XmlChangeset::isMatch(const XmlChangeset& changeset)
+{
+ bool isEqual = true;
+ // Check counts first
+ if (_allNodes.size() != changeset._allNodes.size())
+ {
+ LOG_WARN("Node count not equal (ref: " << _allNodes.size() << ", test: " << changeset._allNodes.size() << ")");
+ isEqual = false;
+ }
+ if (_allWays.size() != changeset._allWays.size())
+ {
+ LOG_WARN("Way count not equal (ref: " << _allWays.size() << ", test: " << changeset._allWays.size() << ")");
+ isEqual = false;
+ }
+ if (_allRelations.size() != changeset._allRelations.size())
+ {
+ LOG_WARN("Relation count not equal (ref: " << _allRelations.size() << ", test: " << changeset._allRelations.size() << ")");
+ isEqual = false;
+ }
+ // Now check each type, create, modify, delete
+ for (int changeset_type = ChangesetType::TypeCreate; changeset_type != ChangesetType::TypeMax; ++changeset_type)
+ {
+ // Iterate all of the nodes of "type" in the changeset
+ QSet<long> missingNodes;
+ for (ChangesetElementMap::iterator it = _nodes[changeset_type].begin(); it != _nodes[changeset_type].end(); ++it)
+ {
+ ChangesetNode* node = dynamic_cast<ChangesetNode*>(it->second.get());
+ ChangesetElementMap::const_iterator found = changeset._nodes[changeset_type].find(node->id());
+ if (found != changeset._nodes[changeset_type].end())
+ {
+ // Compare the two nodes
+ ChangesetNode* node2 = dynamic_cast<ChangesetNode*>(found->second.get());
+ QString output;
+ if (!node->diff(*node2, output))
+ {
+ // Display the node diff
+ output.chop(1);
+ LOG_WARN("Node ID " << node->id() << "\n" << output);
+ }
+ }
+ else
+ missingNodes.insert(node->id());
+ }
+ // Output missing nodes in full
+ if (missingNodes.size() > 0)
+ {
+ QString buffer;
+ QTextStream ts(&buffer);
+ ts.setCodec("UTF-8");
+ for (QSet<long>::iterator it = missingNodes.begin(); it != missingNodes.end(); ++it)
+ ts << _nodes[changeset_type][*it]->toString(0);
+ buffer.chop(1);
+ buffer.replace("\n", "\n>");
+ LOG_WARN("Missing nodes: " << getString(static_cast<ChangesetType>(changeset_type)) <<
+ " - " << missingNodes << "\n>" << buffer);
+ isEqual = false;
+ }
+ // Iterate all of the ways of "type" in the changeset
+ QSet<long> missingWays;
+ for (ChangesetElementMap::iterator it = _ways[changeset_type].begin(); it != _ways[changeset_type].end(); ++it)
+ {
+ ChangesetWay* way = dynamic_cast<ChangesetWay*>(it->second.get());
+ ChangesetElementMap::const_iterator found = changeset._ways[changeset_type].find(way->id());
+ if (found != changeset._ways[changeset_type].end())
+ {
+ // Compare the two ways
+ ChangesetWay* way2 = dynamic_cast<ChangesetWay*>(found->second.get());
+ QString output;
+ if (!way->diff(*way2, output))
+ {
+ // Display the way diff
+ output.chop(1);
+ LOG_WARN("Way ID " << way->id() << "\n" << output);
+ }
+ }
+ else
+ missingWays.insert(way->id());
+ }
+ // Output missing ways in full
+ if (missingWays.size() > 0)
+ {
+ QString buffer;
+ QTextStream ts(&buffer);
+ ts.setCodec("UTF-8");
+ for (QSet<long>::iterator it = missingWays.begin(); it != missingWays.end(); ++it)
+ ts << _ways[changeset_type][*it]->toString(0);
+ buffer.chop(1);
+ buffer.replace("\n", "\n>");
+ LOG_WARN("Missing ways: " << getString(static_cast<ChangesetType>(changeset_type)) <<
+ " - " << missingWays << "\n>" << buffer);
+ isEqual = false;
+ }
+ // Iterate all of the relations of "type" in the changeset
+ QSet<long> missingRelations;
+ for (ChangesetElementMap::iterator it = _relations[changeset_type].begin(); it != _relations[changeset_type].end(); ++it)
+ {
+ ChangesetRelation* relation = dynamic_cast<ChangesetRelation*>(it->second.get());
+ ChangesetElementMap::const_iterator found = changeset._relations[changeset_type].find(relation->id());
+ if (found != changeset._relations[changeset_type].end())
+ {
+ // Compare the two ways
+ ChangesetRelation* relation2 = dynamic_cast<ChangesetRelation*>(found->second.get());
+ QString output;
+ if (!relation->diff(*relation2, output))
+ {
+ // Display the relation diff
+ output.chop(1);
+ LOG_WARN("Relation ID " << relation->id() << "\n" << output);
+ }
+ }
+ else
+ missingRelations.insert(relation->id());
+ }
+ // Output missing relations in full
+ if (missingRelations.size() > 0)
+ {
+ QString buffer;
+ QTextStream ts(&buffer);
+ ts.setCodec("UTF-8");
+ for (QSet<long>::iterator it = missingRelations.begin(); it != missingRelations.end(); ++it)
+ ts << _relations[changeset_type][*it]->toString(0);
+ buffer.chop(1);
+ buffer.replace("\n", "\n>");
+ LOG_WARN("Missing relations: " << getString(static_cast<ChangesetType>(changeset_type)) <<
+ " - " << missingRelations << "\n>" << buffer);
+ isEqual = false;
+ }
+ }
+ return isEqual;
}
ChangesetInfo::ChangesetInfo()