v0.2.47..v0.2.48 changeset OsmXmlReader.cpp
Garret Voltz edited this page Sep 27, 2019
·
1 revision
diff --git a/hoot-core/src/main/cpp/hoot/core/io/OsmXmlReader.cpp b/hoot-core/src/main/cpp/hoot/core/io/OsmXmlReader.cpp
index dbaf405..ec70079 100644
--- a/hoot-core/src/main/cpp/hoot/core/io/OsmXmlReader.cpp
+++ b/hoot-core/src/main/cpp/hoot/core/io/OsmXmlReader.cpp
@@ -43,8 +43,8 @@
#include <hoot/core/util/MapProjector.h>
#include <hoot/core/visitors/ReportMissingElementsVisitor.h>
#include <hoot/core/util/StringUtils.h>
-#include <hoot/core/ops/MapCropper.h>
#include <hoot/core/util/GeometryUtils.h>
+#include <hoot/core/util/IoUtils.h>
// Qt
#include <QBuffer>
@@ -64,9 +64,12 @@ HOOT_FACTORY_REGISTER(OsmMapReader, OsmXmlReader)
OsmXmlReader::OsmXmlReader() :
_status(Status::Invalid),
+_missingNodeCount(0),
+_missingWayCount(0),
_useDataSourceId(false),
_inputCompressed(false),
-_numRead(0)
+_numRead(0),
+_keepImmediatelyConnectedWaysOutsideBounds(false)
{
setConfiguration(conf());
}
@@ -87,6 +90,8 @@ void OsmXmlReader::setConfiguration(const Settings& conf)
setAddChildRefsWhenMissing(configOptions.getOsmMapReaderXmlAddChildRefsWhenMissing());
setStatusUpdateInterval(configOptions.getTaskStatusUpdateInterval() * 10);
setBounds(GeometryUtils::envelopeFromConfigString(configOptions.getConvertBoundingBox()));
+ setKeepImmediatelyConnectedWaysOutsideBounds(
+ ConfigOptions().getConvertBoundingBoxKeepImmediatelyConnectedWaysOutsideBounds());
}
void OsmXmlReader::_parseTimeStamp(const QXmlAttributes &attributes)
@@ -99,10 +104,17 @@ void OsmXmlReader::_parseTimeStamp(const QXmlAttributes &attributes)
}
}
-void OsmXmlReader::_createNode(const QXmlAttributes &attributes)
+void OsmXmlReader::_createNode(const QXmlAttributes& attributes)
{
long id = _parseLong(attributes.value("id"));
//LOG_VART(id);
+
+ if (_nodeIdMap.contains(id))
+ {
+ throw HootException(
+ QString("Duplicate node id %1 in map %2 encountered.").arg(id).arg(_path));
+ }
+
long newId;
if (_useDataSourceId)
{
@@ -118,7 +130,7 @@ void OsmXmlReader::_createNode(const QXmlAttributes &attributes)
double x = _parseDouble(attributes.value("lon"));
double y = _parseDouble(attributes.value("lat"));
- // check the next 3 attributes to see if a value exist, if not, assign a default since these
+ // check the next 3 attributes to see if a value exists, if not, assign a default since these
// are not officially required by the DTD
long version = ElementData::VERSION_EMPTY;
if (attributes.value("version") != "")
@@ -156,13 +168,29 @@ void OsmXmlReader::_createNode(const QXmlAttributes &attributes)
}
}
-void OsmXmlReader::_createRelation(const QXmlAttributes &attributes)
+void OsmXmlReader::_createWay(const QXmlAttributes& attributes)
{
- _relationId = _parseLong(attributes.value("id"));
- long newId = _getRelationId(_relationId);
+ _wayId = _parseLong(attributes.value("id"));
- // check the next 3 attributes to see if a value exist, if not, assign a default since these are
- // not officially required by the DTD
+ if (_wayIdMap.contains(_wayId))
+ {
+ throw HootException(
+ QString("Duplicate way id %1 in map %2 encountered.").arg(_wayId).arg(_path));
+ }
+
+ long newId;
+ if (_useDataSourceId)
+ {
+ newId = _wayId;
+ }
+ else
+ {
+ newId = _map->createNextWayId();
+ }
+ _wayIdMap.insert(_wayId, newId);
+
+ // check the next 3 attributes to see if a value exists, if not, assign a default since
+ // these are not officially required by the DTD
long version = ElementData::VERSION_EMPTY;
if (attributes.value("version") != "")
{
@@ -190,34 +218,26 @@ void OsmXmlReader::_createRelation(const QXmlAttributes &attributes)
}
_element.reset(
- new Relation(
- _status, newId, _defaultCircularError, "", changeset, version, timestamp, user, uid));
+ new Way(_status, newId, _defaultCircularError, changeset, version, timestamp, user, uid));
_parseTimeStamp(attributes);
}
-void OsmXmlReader::_createWay(const QXmlAttributes &attributes)
+void OsmXmlReader::_createRelation(const QXmlAttributes& attributes)
{
- _wayId = _parseLong(attributes.value("id"));
+ _relationId = _parseLong(attributes.value("id"));
- if (_wayIdMap.contains(_wayId))
- {
- throw HootException(QString("Duplicate way id %1 in map %2 encountered.").arg(_wayId).arg(_path));
- }
+ // Adding this in causes issues with the tests...worth looking into at some point.
+// if (_relationIdMap.contains(_relationId))
+// {
+// throw HootException(
+// QString("Duplicate relation id %1 in map %2 encountered.").arg(_relationId).arg(_path));
+// }
- long newId;
- if (_useDataSourceId)
- {
- newId = _wayId;
- }
- else
- {
- newId = _map->createNextWayId();
- }
- _wayIdMap.insert(_wayId, newId);
+ long newId = _getRelationId(_relationId);
- // check the next 3 attributes to see if a value exist, if not, assign a default since
- // these are not officially required by the DTD
+ // check the next 3 attributes to see if a value exists, if not, assign a default since these are
+ // not officially required by the DTD
long version = ElementData::VERSION_EMPTY;
if (attributes.value("version") != "")
{
@@ -245,7 +265,8 @@ void OsmXmlReader::_createWay(const QXmlAttributes &attributes)
}
_element.reset(
- new Way(_status, newId, _defaultCircularError, changeset, version, timestamp, user, uid));
+ new Relation(
+ _status, newId, _defaultCircularError, "", changeset, version, timestamp, user, uid));
_parseTimeStamp(attributes);
}
@@ -318,7 +339,7 @@ void OsmXmlReader::read(const OsmMapPtr& map)
LOG_VART(_keepStatusTag);
LOG_VART(_preserveAllTags);
- // clear node id maps in case the reader is used for mulitple files
+ // clear node id maps in case the reader is used for multiple files
_nodeIdMap.clear();
_relationIdMap.clear();
_wayIdMap.clear();
@@ -351,30 +372,23 @@ void OsmXmlReader::read(const OsmMapPtr& map)
throw HootException(_errorString);
}
file.close();
+ LOG_VARD(StringUtils::formatLargeNumber(_map->getElementCount()));
- // We don't support cropping during streaming, and there is a check in
- // ElementStreamer::isStreamableIo to make sure nothing tries to stream with this reader when
- // a bounds has been set.
+ // This is meant for taking a larger input down to a smaller size. Clearly, if the input data's
+ // bounds is already smaller than _bounds, this will have no effect. Also, We don't support
+ // cropping during streaming, and there is a check in ElementStreamer::isStreamableIo to make
+ // sure nothing tries to stream with this reader when a bounds has been set.
LOG_VARD(_bounds.isNull());
if (!_bounds.isNull())
{
- LOG_INFO("Applying bounds filtering to ingested data: " << _bounds << "...");
- MapCropper cropper(_bounds);
- LOG_INFO(cropper.getInitStatusMessage());
- // We don't reuse MapCropper's version of these options, since we want the freedom to have
- // different default values than what MapCropper uses.
- cropper.setKeepEntireFeaturesCrossingBounds(
- ConfigOptions().getConvertBoundingBoxKeepEntireFeaturesCrossingBounds());
- cropper.setKeepOnlyFeaturesInsideBounds(
- ConfigOptions().getConvertBoundingBoxKeepOnlyFeaturesInsideBounds());
- cropper.apply(_map);
- LOG_INFO(cropper.getCompletedStatusMessage());
+ IoUtils::cropToBounds(_map, _bounds, _keepImmediatelyConnectedWaysOutsideBounds);
+ LOG_VARD(StringUtils::formatLargeNumber(_map->getElementCount()));
}
ReportMissingElementsVisitor visitor;
LOG_INFO("\t" << visitor.getInitStatusMessage());
_map->visitRw(visitor);
- LOG_INFO("\t" << visitor.getCompletedStatusMessage());
+ LOG_DEBUG("\t" << visitor.getCompletedStatusMessage());
_map.reset();
}
@@ -399,6 +413,13 @@ void OsmXmlReader::readFromString(const QString& xml, const OsmMapPtr& map)
throw Exception(_errorString);
}
+ LOG_VARD(_bounds.isNull());
+ if (!_bounds.isNull())
+ {
+ IoUtils::cropToBounds(_map, _bounds, _keepImmediatelyConnectedWaysOutsideBounds);
+ LOG_VARD(StringUtils::formatLargeNumber(_map->getElementCount()));
+ }
+
ReportMissingElementsVisitor visitor;
LOG_INFO("\t" << visitor.getInitStatusMessage());
_map->visitRw(visitor);
@@ -496,7 +517,9 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
_missingNodeCount++;
if (logWarnCount < Log::getWarnMessageLimit())
{
- LOG_WARN("Missing node (" << ref << ") in way (" << _wayId << ").");
+ LOG_WARN(
+ "Missing " << ElementId(ElementType::Node, ref) << " in " <<
+ ElementId(ElementType::Way, _wayId) << ".");
}
else if (logWarnCount == Log::getWarnMessageLimit())
{
@@ -508,7 +531,7 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
else
{
long newRef = _nodeIdMap.value(ref);
- //LOG_TRACE("Adding way node: " << newRef << "...");
+ LOG_TRACE("Adding way node: " << newRef << "...");
WayPtr w = std::dynamic_pointer_cast<Way, Element>(_element);
w->addNode(newRef);
}
@@ -536,7 +559,9 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
_missingNodeCount++;
if (logWarnCount < Log::getWarnMessageLimit())
{
- LOG_WARN("Missing node (" << ref << ") in relation (" << _relationId << ").");
+ LOG_WARN(
+ "Missing " << ElementId(ElementType::Node, ref) << " in " <<
+ ElementId(ElementType::Relation, _relationId) << ".");
}
else if (logWarnCount == Log::getWarnMessageLimit())
{
@@ -548,7 +573,7 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
else
{
long newRef = _nodeIdMap.value(ref);
- //LOG_TRACE("Adding relation node member: " << newRef << "...");
+ LOG_TRACE("Adding relation node member: " << newRef << "...");
r->addElement(role, ElementType::Node, newRef);
}
}
@@ -567,7 +592,9 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
_missingWayCount++;
if (logWarnCount < Log::getWarnMessageLimit())
{
- LOG_WARN("Missing way (" << ref << ") in relation (" << _relationId << ").");
+ LOG_WARN(
+ "Missing " << ElementId(ElementType::Way, ref) << " in " <<
+ ElementId(ElementType::Relation, _relationId) << ".");
}
else if (logWarnCount == Log::getWarnMessageLimit())
{
@@ -579,7 +606,7 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
else
{
long newRef = _wayIdMap.value(ref);
- //LOG_TRACE("Adding relation way member: " << newRef << "...");
+ LOG_TRACE("Adding relation way member: " << newRef << "...");
r->addElement(role, ElementType::Way, newRef);
}
}
@@ -587,7 +614,7 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
{
// relations may be out of order so we don't check for consistency at this stage.
long newRef = _getRelationId(ref);
- //LOG_TRACE("Adding relation relation member: " << newRef << "...");
+ LOG_TRACE("Adding relation relation member: " << newRef << "...");
r->addElement(role, ElementType::Relation, newRef);
}
else
@@ -608,8 +635,8 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
{
const QString& key = _saveMemory(attributes.value("k").trimmed());
const QString& value = _saveMemory(attributes.value("v").trimmed());
- //LOG_VART(key);
- //LOG_VART(value);
+ LOG_VART(key);
+ LOG_VART(value);
if (!key.isEmpty() && !value.isEmpty())
{
if (_useFileStatus && key == MetadataTags::HootStatus())
@@ -674,7 +701,7 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
}
if (_preserveAllTags)
{
- //LOG_TRACE("setting tag with key: " << key << " and value: " << value);
+ LOG_TRACE("setting tag with key: " << key << " and value: " << value);
_element->setTag(key, value);
}
}
@@ -682,7 +709,7 @@ bool OsmXmlReader::startElement(const QString& /*namespaceURI*/, const QString&
{
if (key != MetadataTags::HootId() && !value.isEmpty())
{
- //LOG_TRACE("setting tag with key: " << key << " and value: " << value);
+ LOG_TRACE("setting tag with key: " << key << " and value: " << value);
_element->setTag(key, value);
}
}
@@ -708,21 +735,24 @@ bool OsmXmlReader::endElement(const QString& /* namespaceURI */,
{
NodePtr n = std::dynamic_pointer_cast<Node, Element>(_element);
_map->addNode(n);
- //LOG_VART(n);
+ LOG_TRACE("Added: " << n->getElementId());
+ //LOG_TRACE("Added: " << n);
_numRead++;
}
else if (qName == QLatin1String("way"))
{
WayPtr w = std::dynamic_pointer_cast<Way, Element>(_element);
_map->addWay(w);
- //LOG_VART(w);
+ LOG_TRACE("Added: " << w->getElementId());
+ //LOG_TRACE("Added: " << w);
_numRead++;
}
else if (qName == QLatin1String("relation"))
{
RelationPtr r = std::dynamic_pointer_cast<Relation, Element>(_element);
_map->addRelation(r);
- //LOG_VART(r);
+ LOG_TRACE("Added: " << r->getElementId());
+ //LOG_TRACE("Added: " << r);
_numRead++;
}