v0.2.54..v0.2.55 changeset ScriptMatchCreator.cpp
Garret Voltz edited this page Aug 14, 2020
·
1 revision
diff --git a/hoot-js/src/main/cpp/hoot/js/conflate/matching/ScriptMatchCreator.cpp b/hoot-js/src/main/cpp/hoot/js/conflate/matching/ScriptMatchCreator.cpp
index 60849a2..5be6b03 100644
--- a/hoot-js/src/main/cpp/hoot/js/conflate/matching/ScriptMatchCreator.cpp
+++ b/hoot-js/src/main/cpp/hoot/js/conflate/matching/ScriptMatchCreator.cpp
@@ -46,6 +46,8 @@
#include <hoot/js/elements/OsmMapJs.h>
#include <hoot/js/elements/ElementJs.h>
#include <hoot/core/criterion/ChainCriterion.h>
+#include <hoot/core/util/MemoryUsageChecker.h>
+#include <hoot/core/elements/ElementConverter.h>
// Qt
#include <QFileInfo>
@@ -98,6 +100,7 @@ public:
_numElementsVisited(0),
_numMatchCandidatesVisited(0),
_taskStatusUpdateInterval(ConfigOptions().getTaskStatusUpdateInterval()),
+ _memoryCheckUpdateInterval(ConfigOptions().getMemoryUsageCheckerInterval()),
_totalElementsToProcess(0)
{
// Calls to script functions/var are expensive, both memory-wise and processing-wise. Since this
@@ -330,7 +333,10 @@ public:
void calculateSearchRadius()
{
- // This is meant to run one time when the match creator is initialized.
+ // This is meant to run one time when the match creator is initialized. We're either getting
+ // the search radius from a predefined property linked to a config option in the conflate rules
+ // file or we're calculating it with a function call from the rules file. Either way, then we're
+ // caching it after the first retrieval.
LOG_DEBUG("Checking for existence of search radius export for: " << _scriptPath << "...");
@@ -340,7 +346,7 @@ public:
Persistent<Object> plugin(current, getPlugin(_script));
Handle<String> initStr = String::NewFromUtf8(current, "calculateSearchRadius");
- //optional method, so don't throw an error
+ // optional method, so don't throw an error
if (ToLocal(&plugin)->Has(initStr) == false)
{
LOG_TRACE("calculateSearchRadius function not present.");
@@ -353,19 +359,19 @@ public:
return;
}
- LOG_STATUS("Calculating search radius for: " << _scriptPath << "...");
+ LOG_DEBUG("Getting search radius for: " << _scriptPath << "...");
Handle<Function> func = Handle<Function>::Cast(value);
Handle<Value> jsArgs[1];
int argc = 0;
- HandleScope scope(current); // This extra one might not be needed
+ HandleScope scope(current); // This extra one might not be needed
assert(getMap().get());
OsmMapPtr copiedMap(new OsmMap(getMap()));
jsArgs[argc++] = OsmMapJs::create(copiedMap);
func->Call(ToLocal(&plugin), argc, jsArgs);
- //this is meant to have been set externally in a js rules file
+ // This could be an expensive call.
_customSearchRadius =
getNumber(
ToLocal(&plugin), "searchRadius", -1.0, ConfigOptions().getCircularErrorDefaultValue());
@@ -509,60 +515,30 @@ public:
bool result = false;
- // Prioritize exports.matchCandidateCriterion over the isMatchCandidate function
- // TODO: this is crashing; see #3047
-// Handle<String> matchCandidateCriterionStrHandle =
-// String::NewFromUtf8(current, "matchCandidateCriterion");
-// QString matchCandidateCriterionStr;
-// if (ToLocal(&plugin)->Has(matchCandidateCriterionStrHandle))
-// {
-// Handle<Value> value = ToLocal(&plugin)->Get(matchCandidateCriterionStrHandle);
-// matchCandidateCriterionStr = toCpp<QString>(value);
-// }
-// matchCandidateCriterionStr = matchCandidateCriterionStr.trimmed();
-// LOG_VART(matchCandidateCriterionStr);
-
-// if (!matchCandidateCriterionStr.isEmpty())
-// {
-// std::shared_ptr<ElementCriterion> matchCandidateCriterion;
-// if (_matchCandidateCriterionCache.contains(matchCandidateCriterionStr))
-// {
-// LOG_TRACE("Getting " << matchCandidateCriterionStr << " from cache...");
-// matchCandidateCriterion = _matchCandidateCriterionCache[matchCandidateCriterionStr];
-// }
-// else
-// {
-// LOG_TRACE("Creating " << matchCandidateCriterionStr << "...");
-// matchCandidateCriterion.reset(
-// Factory::getInstance().constructObject<ElementCriterion>(matchCandidateCriterionStr));
-// _matchCandidateCriterionCache[matchCandidateCriterionStr] = matchCandidateCriterion;
-// }
-// result = matchCandidateCriterion->isSatisfied(e);
-// LOG_VART(result);
-// }
-// else
-// {
- Handle<String> isMatchCandidateStr = String::NewFromUtf8(current, "isMatchCandidate");
- if (ToLocal(&plugin)->Has(isMatchCandidateStr) == false)
- {
- throw HootException("Error finding 'isMatchCandidate' function.");
- }
- Handle<Value> value = ToLocal(&plugin)->Get(isMatchCandidateStr);
- if (value->IsFunction() == false)
- {
- throw HootException("isMatchCandidate is not a function.");
- }
- Handle<Function> func = Handle<Function>::Cast(value);
- Handle<Value> jsArgs[2];
+ // TODO: Prioritize exports.matchCandidateCriterion over the isMatchCandidate function and use
+ // the crit instead of the function; doing so causes this to crash; see #3047 and the history
+ // of this file for the failing code that needs to be re-enabled
- int argc = 0;
- jsArgs[argc++] = getOsmMapJs();
- jsArgs[argc++] = ElementJs::New(e);
+ Handle<String> isMatchCandidateStr = String::NewFromUtf8(current, "isMatchCandidate");
+ if (ToLocal(&plugin)->Has(isMatchCandidateStr) == false)
+ {
+ throw HootException("Error finding 'isMatchCandidate' function.");
+ }
+ Handle<Value> value = ToLocal(&plugin)->Get(isMatchCandidateStr);
+ if (value->IsFunction() == false)
+ {
+ throw HootException("isMatchCandidate is not a function.");
+ }
+ Handle<Function> func = Handle<Function>::Cast(value);
+ Handle<Value> jsArgs[2];
- Handle<Value> f = func->Call(ToLocal(&plugin), argc, jsArgs);
+ int argc = 0;
+ jsArgs[argc++] = getOsmMapJs();
+ jsArgs[argc++] = ElementJs::New(e);
- result = f->BooleanValue();
- //}
+ Handle<Value> f = func->Call(ToLocal(&plugin), argc, jsArgs);
+
+ result = f->BooleanValue();
_matchCandidateCache[e->getElementId()] = result;
@@ -580,7 +556,7 @@ public:
{
PROGRESS_DEBUG(
"Processed " << StringUtils::formatLargeNumber(_numMatchCandidatesVisited) <<
- " match candidates / " << StringUtils::formatLargeNumber(getMap()->getElementCount()) <<
+ " match candidates / " << StringUtils::formatLargeNumber(_totalElementsToProcess) <<
" total elements.");
}
}
@@ -603,6 +579,10 @@ public:
StringUtils::formatLargeNumber(_totalElementsToProcess) << " elements.");
_timer.restart();
}
+ if (_numElementsVisited % _memoryCheckUpdateInterval == 0)
+ {
+ MemoryUsageChecker::getInstance().check();
+ }
}
void setScriptPath(QString path) { _scriptPath = path; }
@@ -619,21 +599,28 @@ public:
{
_scriptInfo = description;
- switch (_scriptInfo.geometryType)
+ if (_scriptPath.toLower().contains("relation")) // hack
{
- case GeometryTypeCriterion::GeometryType::Point:
- _totalElementsToProcess = getMap()->getNodeCount();
- break;
- case GeometryTypeCriterion::GeometryType::Line:
- _totalElementsToProcess = getMap()->getWayCount() + getMap()->getRelationCount();
- break;
- case GeometryTypeCriterion::GeometryType::Polygon:
- _totalElementsToProcess = getMap()->getWayCount() + getMap()->getRelationCount();
- break;
- default:
- // visit all geometry types if the script didn't identify its geometry
- _totalElementsToProcess = getMap()->size();
- break;
+ _totalElementsToProcess = getMap()->getRelationCount();
+ }
+ else
+ {
+ switch (_scriptInfo.geometryType)
+ {
+ case GeometryTypeCriterion::GeometryType::Point:
+ _totalElementsToProcess = getMap()->getNodeCount();
+ break;
+ case GeometryTypeCriterion::GeometryType::Line:
+ _totalElementsToProcess = getMap()->getWayCount() + getMap()->getRelationCount();
+ break;
+ case GeometryTypeCriterion::GeometryType::Polygon:
+ _totalElementsToProcess = getMap()->getWayCount() + getMap()->getRelationCount();
+ break;
+ default:
+ // visit all geometry types if the script didn't identify its geometry
+ _totalElementsToProcess = getMap()->size();
+ break;
+ }
}
}
@@ -654,8 +641,8 @@ private:
ElementCriterionPtr _filter;
- //used for automatic search radius calculation; it is expected that this is set from the
- //Javascript rules file used for the generic conflation
+ // used for automatic search radius calculation; it is expected that this is set from the
+ // Javascript rules file used for the generic conflation
double _customSearchRadius;
int _neighborCountMax;
@@ -666,6 +653,7 @@ private:
long _numMatchCandidatesVisited;
int _taskStatusUpdateInterval;
+ int _memoryCheckUpdateInterval;
long _totalElementsToProcess;
@@ -842,24 +830,32 @@ void ScriptMatchCreator::createMatches(
_candidateDistanceSigmaCache[_scriptPath] = v.getCandidateDistanceSigma();
LOG_VARD(GeometryTypeCriterion::typeToString(_scriptInfo.geometryType));
- switch (_scriptInfo.geometryType)
+ if (scriptFileInfo.fileName().toLower().contains("relation")) // hack
{
- case GeometryTypeCriterion::GeometryType::Point:
- map->visitNodesRo(v);
- break;
- case GeometryTypeCriterion::GeometryType::Line:
- map->visitWaysRo(v);
- map->visitRelationsRo(v);
- break;
- case GeometryTypeCriterion::GeometryType::Polygon:
- map->visitWaysRo(v);
- map->visitRelationsRo(v);
- break;
- default:
- // visit all geometry types if the script didn't identify its geometry
- map->visitRo(v);
- break;
+ map->visitRelationsRo(v);
}
+ else
+ {
+ switch (_scriptInfo.geometryType)
+ {
+ case GeometryTypeCriterion::GeometryType::Point:
+ map->visitNodesRo(v);
+ break;
+ case GeometryTypeCriterion::GeometryType::Line:
+ map->visitWaysRo(v);
+ map->visitRelationsRo(v);
+ break;
+ case GeometryTypeCriterion::GeometryType::Polygon:
+ map->visitWaysRo(v);
+ map->visitRelationsRo(v);
+ break;
+ default:
+ // visit all geometry types if the script didn't identify its geometry
+ map->visitRo(v);
+ break;
+ }
+ }
+
const int matchesSizeAfter = matches.size();
QString matchType = CreatorDescription::baseFeatureTypeToString(_scriptInfo.baseFeatureType);