v0.2.52..v0.2.53 changeset GrailResource.java
Garret Voltz edited this page Feb 12, 2020
·
1 revision
diff --git a/hoot-services/src/main/java/hoot/services/controllers/grail/GrailResource.java b/hoot-services/src/main/java/hoot/services/controllers/grail/GrailResource.java
index 2849e23..969387a 100644
--- a/hoot-services/src/main/java/hoot/services/controllers/grail/GrailResource.java
+++ b/hoot-services/src/main/java/hoot/services/controllers/grail/GrailResource.java
@@ -22,11 +22,12 @@
* This will properly maintain the copyright information. DigitalGlobe
* copyrights will be updated automatically.
*
- * @copyright Copyright (C) 2016, 2017, 2018, 2019 DigitalGlobe (http://www.digitalglobe.com/)
+ * @copyright Copyright (C) 2016, 2017, 2018, 2019, 2020 DigitalGlobe (http://www.digitalglobe.com/)
*/
package hoot.services.controllers.grail;
import static hoot.services.HootProperties.CHANGESETS_FOLDER;
+import static hoot.services.HootProperties.CHANGESET_OPTIONS;
import static hoot.services.HootProperties.GRAIL_OVERPASS_LABEL;
import static hoot.services.HootProperties.GRAIL_OVERPASS_STATS_QUERY;
import static hoot.services.HootProperties.GRAIL_RAILS_LABEL;
@@ -45,12 +46,12 @@ import static hoot.services.HootProperties.replaceSensitiveData;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.URL;
-import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
@@ -87,6 +88,7 @@ import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.commons.lang3.StringUtils;
import org.apache.xpath.XPathAPI;
import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -144,6 +146,9 @@ public class GrailResource {
@Autowired
private UpdateParentCommandFactory updateParentCommandFactory;
+ @Autowired
+ private PullConnectedWaysCommandFactory connectedWaysCommandFactory;
+
public GrailResource() {}
private Command getRailsPortApiCommand(String jobId, GrailParams params) throws UnavailableException {
@@ -165,6 +170,13 @@ public class GrailResource {
return command;
}
+ private Command getConnectedWaysApiCommand(String jobId, GrailParams params) throws UnavailableException {
+ params.setPullUrl(RAILSPORT_PULL_URL);
+
+ InternalCommand command = connectedWaysCommandFactory.build(jobId, params, this.getClass());
+ return command;
+ }
+
private Command getPublicOverpassCommand(String jobId, GrailParams params) {
params.setPullUrl(PUBLIC_OVERPASS_URL);
@@ -240,7 +252,7 @@ public class GrailResource {
workflow.add(getPublicOverpassCommand(jobId, getOverpassParams));
// Run the differential conflate command.
- GrailParams params = new GrailParams();
+ GrailParams params = new GrailParams(reqParams);
params.setUser(user);
params.setInput1(referenceOSMFile.getAbsolutePath());
params.setInput2(secondaryOSMFile.getAbsolutePath());
@@ -443,30 +455,6 @@ public class GrailResource {
}
}
- Map<String, String> tags = DbUtils.getJobTags(reqParams.getParentId());
- String resourceId = tags.get("input1");
-
- if(resourceId != null) {
- // Setup workflow to refresh rails data after the push
- long referenceId = Long.parseLong(resourceId);
- Long parentFolderId = DbUtils.getParentFolder(referenceId);
- Map<String, String> mapTags = DbUtils.getMapsTableTags(referenceId);
-
- GrailParams refreshParams = new GrailParams();
- refreshParams.setUser(user);
- refreshParams.setWorkDir(workDir);
- refreshParams.setOutput(DbUtils.getDisplayNameById(referenceId));
- refreshParams.setBounds(mapTags.get("bbox"));
-
- try {
- List<Command> refreshWorkflow = setupRailsPull(jobId, refreshParams, parentFolderId);
- workflow.addAll(refreshWorkflow);
- }
- catch(UnavailableException exc) {
- return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(exc.getMessage()).build();
- }
- }
-
Map<String, Object> jobStatusTags = new HashMap<>();
jobStatusTags.put("bbox", reqParams.getBounds());
jobStatusTags.put("parentId", reqParams.getParentId());
@@ -613,6 +601,7 @@ public class GrailResource {
String bbox = reqParams.getBounds();
String layerName = reqParams.getInput1();
String jobId = UUID.randomUUID().toString().replace("-", "");
+ File workDir = new File(TEMP_OUTPUT_PATH, "grail_" + jobId);
if (DbUtils.mapExists(layerName)) {
throw new BadRequestException("Record with name : " + layerName + " already exists. Please try a different name.");
@@ -635,14 +624,22 @@ public class GrailResource {
if (customQuery == null || customQuery.equals("")) {
url = "'" + PullOverpassCommand.getOverpassUrl(bbox) + "'";
} else {
- url = "'" + PullOverpassCommand.getOverpassUrl(replaceSensitiveData(params.getPullUrl()), bbox, "json", customQuery) + "'";
+ url = "'" + PullOverpassCommand.getOverpassUrl(replaceSensitiveData(params.getPullUrl()), bbox, "xml", customQuery) + "'";
}
} catch(IllegalArgumentException exc) {
return Response.status(Response.Status.BAD_REQUEST).entity(exc.getMessage()).build();
}
- params.setInput1(url);
+
+ File overpassOSMFile = new File(workDir, SECONDARY + ".osm");
+ GrailParams getOverpassParams = new GrailParams(params);
+ getOverpassParams.setOutput(overpassOSMFile.getAbsolutePath());
+ if (overpassOSMFile.exists()) overpassOSMFile.delete();
+ workflow.add(getPublicOverpassCommand(jobId, getOverpassParams));
+
+
+ params.setInput1(overpassOSMFile.getAbsolutePath());
params.setOutput(layerName);
ExternalCommand importOverpass = grailCommandFactory.build(jobId, params, "info", PushToDbCommand.class, this.getClass());
workflow.add(importOverpass);
@@ -819,13 +816,10 @@ public class GrailResource {
advancedUserCheck(user);
String layerName = reqParams.getInput1();
+
String jobId = UUID.randomUUID().toString().replace("-", "");
File workDir = new File(TEMP_OUTPUT_PATH, "grail_" + jobId);
- if (DbUtils.mapExists(layerName)) {
- throw new BadRequestException("Record with name : " + layerName + " already exists. Please try a different name.");
- }
-
JSONObject json = new JSONObject();
json.put("jobid", jobId);
@@ -869,10 +863,28 @@ public class GrailResource {
throw new UnavailableException("The Rails port API is offline.");
}
- params.setInput1(referenceOSMFile.getAbsolutePath());
-
+ GrailParams connectedWaysParams = new GrailParams(params);
+ connectedWaysParams.setInput1(referenceOSMFile.getAbsolutePath());
+ File cropFile = new File(params.getWorkDir(), "crop.osm");
+ connectedWaysParams.setOutput(cropFile.getAbsolutePath());
+ // Do an invert crop of this data to get nodes outside bounds
+ workflow.add(grailCommandFactory.build(jobId, connectedWaysParams, "info", InvertCropCommand.class, this.getClass()));
+
+ //read node ids
+ //pull connected ways
+ //pull entire ways
+ //remove cropfile
+ workflow.add(getConnectedWaysApiCommand(jobId, connectedWaysParams));
+
+ // merge reference and ways osm files
+ GrailParams mergeOsmParams = new GrailParams(params);
+ File mergeFile = new File(params.getWorkDir(), "merge.osm");
+ mergeOsmParams.setOutput(mergeFile.getAbsolutePath());
+ workflow.add(grailCommandFactory.build(jobId, mergeOsmParams, "info", MergeOsmFilesCommand.class, this.getClass()));
// Write the data to the hoot db
- ExternalCommand importRailsPort = grailCommandFactory.build(jobId, params, "info", PushToDbCommand.class, this.getClass());
+ GrailParams pushParams = new GrailParams(params);
+ pushParams.setInput1(mergeFile.getAbsolutePath());
+ ExternalCommand importRailsPort = grailCommandFactory.build(jobId, pushParams, "info", PushToDbCommand.class, this.getClass());
workflow.add(importRailsPort);
// Set map tags marking dataset as eligible for derive changeset
@@ -980,8 +992,7 @@ public class GrailResource {
if (usePrivateOverpass && !replaceSensitiveData(PRIVATE_OVERPASS_CERT_PATH).equals(PRIVATE_OVERPASS_CERT_PATH)) {
inputStream = PullApiCommand.getHttpResponseWithSSL(url);
} else {
- URLConnection conn = new URL(url).openConnection();
- inputStream = conn.getInputStream();
+ inputStream = PullOverpassCommand.getOverpassInputStream(url);
}
BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
@@ -1017,4 +1028,20 @@ public class GrailResource {
return statCounts;
}
+ @GET
+ @Path("/getChangesetOptions")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getOptions() {
+ JSONObject template;
+ JSONParser parser = new JSONParser();
+ try (FileReader fileReader = new FileReader(new File(HOME_FOLDER, CHANGESET_OPTIONS))) {
+ template = (JSONObject) parser.parse(fileReader);
+ }
+ catch (Exception e) {
+ String msg = "Error getting changeset options! Cause: " + e.getMessage();
+ throw new WebApplicationException(e, Response.serverError().entity(msg).build());
+ }
+ return Response.ok(template).build();
+ }
+
}