v0.2.49..v0.2.50 changeset GrailResource.java
Garret Voltz edited this page Nov 6, 2019
·
1 revision
diff --git a/hoot-services/src/main/java/hoot/services/controllers/grail/GrailResource.java b/hoot-services/src/main/java/hoot/services/controllers/grail/GrailResource.java
index 478753c..797cd0c 100644
--- a/hoot-services/src/main/java/hoot/services/controllers/grail/GrailResource.java
+++ b/hoot-services/src/main/java/hoot/services/controllers/grail/GrailResource.java
@@ -26,13 +26,14 @@
*/
package hoot.services.controllers.grail;
-import static hoot.services.HootProperties.GRAIL_OVERPASS_QUERY;
+import static hoot.services.HootProperties.GRAIL_OVERPASS_LABEL;
import static hoot.services.HootProperties.GRAIL_OVERPASS_STATS_QUERY;
-import static hoot.services.HootProperties.GRAIL_OVERPASS_CODENAME;
-import static hoot.services.HootProperties.GRAIL_RAILS_CODENAME;
+import static hoot.services.HootProperties.GRAIL_RAILS_LABEL;
import static hoot.services.HootProperties.HOME_FOLDER;
import static hoot.services.HootProperties.HOOTAPI_DB_URL;
import static hoot.services.HootProperties.MAX_OVERPASS_FEATURE_COUNT;
+import static hoot.services.HootProperties.PRIVATE_OVERPASS_CERT_PATH;
+import static hoot.services.HootProperties.PRIVATE_OVERPASS_URL;
import static hoot.services.HootProperties.PUBLIC_OVERPASS_URL;
import static hoot.services.HootProperties.RAILSPORT_CAPABILITIES_URL;
import static hoot.services.HootProperties.RAILSPORT_PULL_URL;
@@ -40,11 +41,15 @@ import static hoot.services.HootProperties.RAILSPORT_PUSH_URL;
import static hoot.services.HootProperties.TEMP_OUTPUT_PATH;
import static hoot.services.HootProperties.replaceSensitiveData;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.URL;
+import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.LinkedList;
@@ -137,30 +142,28 @@ public class GrailResource {
public GrailResource() {}
- private Command getRailsPortApiCommand(String jobId, Users user, String bounds, String output) throws UnavailableException {
- APICapabilities railsPortCapabilities = getCapabilities(RAILSPORT_CAPABILITIES_URL);
- if (railsPortCapabilities.getApiStatus() == null
- || railsPortCapabilities.getApiStatus().equals("offline")) {
- throw new UnavailableException("The Rails port API is offline.");
+ private Command getRailsPortApiCommand(String jobId, GrailParams params) throws UnavailableException {
+ // Checks to see that the sensitive data was actually replaced meaning there was a value
+ if (!replaceSensitiveData(PRIVATE_OVERPASS_URL).equals(PRIVATE_OVERPASS_URL)) {
+ params.setPullUrl(PRIVATE_OVERPASS_URL);
+ } else {
+ APICapabilities railsPortCapabilities = getCapabilities(RAILSPORT_CAPABILITIES_URL);
+ if (railsPortCapabilities.getApiStatus() == null
+ || railsPortCapabilities.getApiStatus().equals("offline")) {
+ throw new UnavailableException("The Rails port API is offline.");
+ }
+
+ params.setMaxBBoxSize(railsPortCapabilities.getMaxArea());
+ params.setPullUrl(RAILSPORT_PULL_URL);
}
- GrailParams params = new GrailParams();
- params.setUser(user);
- params.setBounds(bounds);
- params.setMaxBBoxSize(railsPortCapabilities.getMaxArea());
- params.setPullUrl(RAILSPORT_PULL_URL);
- params.setOutput(output);
InternalCommand command = apiCommandFactory.build(jobId, params, this.getClass());
return command;
}
- private Command getPublicOverpassCommand(String jobId, Users user, String bounds, String output) {
- //TODO: is there an availability check for overpass?
- GrailParams params = new GrailParams();
- params.setUser(user);
- params.setBounds(bounds);
+ private Command getPublicOverpassCommand(String jobId, GrailParams params) {
params.setPullUrl(PUBLIC_OVERPASS_URL);
- params.setOutput(output);
+
InternalCommand command = overpassCommandFactory.build(jobId, params, this.getClass());
return command;
}
@@ -194,6 +197,7 @@ public class GrailResource {
Users user = Users.fromRequest(request);
advancedUserCheck(user);
+ reqParams.setUser(user);
String jobId = "grail_" + UUID.randomUUID().toString().replace("-", "");
@@ -207,24 +211,29 @@ public class GrailResource {
}
List<Command> workflow = new LinkedList<>();
- String bbox = reqParams.getBounds();
JSONObject jobInfo = new JSONObject();
jobInfo.put("jobid", jobId);
// Pull reference data from Rails port OSM API
File referenceOSMFile = new File(workDir, REFERENCE + ".osm");
+ GrailParams getRailsParams = new GrailParams(reqParams);
+ getRailsParams.setOutput(referenceOSMFile.getAbsolutePath());
+
if (referenceOSMFile.exists()) referenceOSMFile.delete();
try {
- workflow.add(getRailsPortApiCommand(jobId, user, bbox, referenceOSMFile.getAbsolutePath()));
+ workflow.add(getRailsPortApiCommand(jobId, getRailsParams));
} catch (UnavailableException ex) {
return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(ex.getMessage()).build();
}
// Pull secondary data from the Public Overpass API
File secondaryOSMFile = new File(workDir, SECONDARY + ".osm");
+ GrailParams getOverpassParams = new GrailParams(reqParams);
+ getOverpassParams.setOutput(secondaryOSMFile.getAbsolutePath());
+
if (secondaryOSMFile.exists()) secondaryOSMFile.delete();
- workflow.add(getPublicOverpassCommand(jobId, user, bbox, secondaryOSMFile.getAbsolutePath()));
+ workflow.add(getPublicOverpassCommand(jobId, getOverpassParams));
// Run the differential conflate command.
GrailParams params = new GrailParams();
@@ -428,6 +437,7 @@ public class GrailResource {
if(resourceId != null) {
// Setup workflow to refresh rails data after the push
long referenceId = DbUtils.getMergedReference(resourceId);
+ Long parentFolderId = DbUtils.getParentFolder(referenceId);
Map<String, String> mapTags = DbUtils.getMapsTableTags(referenceId);
GrailParams refreshParams = new GrailParams();
@@ -435,10 +445,9 @@ public class GrailResource {
refreshParams.setWorkDir(workDir);
refreshParams.setOutput(DbUtils.getDisplayNameById(referenceId));
refreshParams.setBounds(mapTags.get("bbox"));
- refreshParams.setParentId("grail_" + mapTags.get("bbox").replace(",", "_"));
try {
- List<Command> refreshWorkflow = setupRailsPull(jobId, refreshParams);
+ List<Command> refreshWorkflow = setupRailsPull(jobId, refreshParams, parentFolderId);
workflow.addAll(refreshWorkflow);
}
catch(UnavailableException exc) {
@@ -556,24 +565,26 @@ public class GrailResource {
* a private OSM instance that has diverged from the public OSM
* with private changes.
*
- * @param bbox The bounding box
+ * @param reqParams Contains info such as bbox, layerName, and if the
+ * user provided one, a custom query for pulling overpass data
*
* @return Job ID
* used by the client for polling job status
* Internally, this provides the map dataset name suffix
*/
- @GET
+ @POST
@Path("/pulloverpasstodb")
@Produces(MediaType.APPLICATION_JSON)
public Response pullOverpassToDb(@Context HttpServletRequest request,
- @QueryParam("bbox") String bbox,
- @QueryParam("name") String layerName) {
+ @QueryParam("folderId") Long folderId,
+ GrailParams reqParams) {
Users user = Users.fromRequest(request);
advancedUserCheck(user);
+ String bbox = reqParams.getBounds();
+ String layerName = reqParams.getInput1();
String jobId = UUID.randomUUID().toString().replace("-", "");
- String folderName = "grail_" + bbox.replace(",", "_");
if (DbUtils.mapExists(layerName)) {
throw new BadRequestException("Record with name : " + layerName + " already exists. Please try a different name.");
@@ -585,16 +596,20 @@ public class GrailResource {
List<Command> workflow = new LinkedList<>();
- // Create the folder if it doesn't exist
- Long folderId = DbUtils.createFolder(folderName, 0L, user.getId(), false);
-
// Write the data to the hoot db
GrailParams params = new GrailParams();
params.setUser(user);
+ params.setPullUrl(PUBLIC_OVERPASS_URL);
String url;
try {
- url = "'" + PullOverpassCommand.getOverpassUrl(bbox) + "'";
+ String customQuery = reqParams.getCustomQuery();
+ if (customQuery == null || customQuery.equals("")) {
+ url = "'" + PullOverpassCommand.getOverpassUrl(bbox) + "'";
+ } else {
+ url = "'" + PullOverpassCommand.getOverpassUrl(replaceSensitiveData(params.getPullUrl()), bbox, "json", customQuery) + "'";
+ }
+
} catch(IllegalArgumentException exc) {
return Response.status(Response.Status.BAD_REQUEST).entity(exc.getMessage()).build();
}
@@ -619,36 +634,78 @@ public class GrailResource {
@GET
@Path("/grailMetadataQuery")
@Produces(MediaType.APPLICATION_JSON)
- public Response grailMetadata(@Context HttpServletRequest request,
- @QueryParam("bbox") String bbox) {
+ public Response grailMetadata(@Context HttpServletRequest request) {
+
+ Users user = Users.fromRequest(request);
+ advancedUserCheck(user);
+
+ String railsLabel = GRAIL_RAILS_LABEL;
+ String overpassLabel = GRAIL_OVERPASS_LABEL;
+
+ JSONObject jobInfo = new JSONObject();
+ jobInfo.put("maxFeatureCount", MAX_OVERPASS_FEATURE_COUNT);
+ jobInfo.put("railsLabel", railsLabel);
+ jobInfo.put("overpassLabel", overpassLabel);
+
+ return Response.ok(jobInfo.toJSONString()).build();
+ }
+
+ @POST
+ @Path("/overpassStats")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response overpassStats(@Context HttpServletRequest request,
+ GrailParams reqParams) {
Users user = Users.fromRequest(request);
advancedUserCheck(user);
+ String customQuery = reqParams.getCustomQuery();
+
// Get grail overpass query from the file and store it in a string
String overpassQuery;
- File overpassQueryFile = new File(HOME_FOLDER, GRAIL_OVERPASS_STATS_QUERY);
- try {
- overpassQuery = FileUtils.readFileToString(overpassQueryFile, "UTF-8");
- } catch(Exception exc) {
- String msg = "Failed to poll overpass for stats query. Couldn't read overpass query file: " + overpassQueryFile.getName();
- throw new WebApplicationException(exc, Response.serverError().entity(msg).build());
+ if (customQuery == null || customQuery.equals("")) {
+ File overpassQueryFile = new File(HOME_FOLDER, GRAIL_OVERPASS_STATS_QUERY);
+ try {
+ overpassQuery = FileUtils.readFileToString(overpassQueryFile, "UTF-8");
+ } catch(Exception exc) {
+ String msg = "Failed to poll overpass for stats query. Couldn't read overpass query file: " + overpassQueryFile.getName();
+ throw new WebApplicationException(exc, Response.serverError().entity(msg).build());
+ }
+ } else {
+ overpassQuery = customQuery;
+
+ if (overpassQuery.contains("out:xml")) {
+ overpassQuery = overpassQuery.replace("out:xml", "out:json");
+ }
+
+ // first line that lists columns which are counts for each feature type
+ overpassQuery = overpassQuery.replace("[out:json]", "[out:csv(::count, ::\"count:nodes\", ::\"count:ways\", ::\"count:relations\")]");
+
+ // last row that lists output format
+ overpassQuery = overpassQuery.replace("out meta", "out count");
}
- //replace the {{bbox}} from the overpass query with the actual coordinates and encode the query
- overpassQuery = overpassQuery.replace("{{bbox}}", new BoundingBox(bbox).toOverpassString());
- String url = replaceSensitiveData(PUBLIC_OVERPASS_URL) + "/api/interpreter?data=" + overpassQuery;
- // append first 7 digits of a uuid to the rails and overpass codenames
- String maxSuffix = UUID.randomUUID().toString().replace("-", "").substring(0, 7);
- String railsCodename = GRAIL_RAILS_CODENAME + "_" + maxSuffix;
- String overpassCodename = GRAIL_OVERPASS_CODENAME + "_" + maxSuffix;
+ //replace the {{bbox}} from the overpass query with the actual coordinates and encode the query
+ overpassQuery = overpassQuery.replace("{{bbox}}", new BoundingBox(reqParams.getBounds()).toOverpassString());
+ try {
+ overpassQuery = URLEncoder.encode(overpassQuery, "UTF-8").replace("+", "%20"); // need to encode url for the get
+ } catch (UnsupportedEncodingException ignored) {} // Can be safely ignored because UTF-8 is always supported
+
+ // Get public overpass data
+ String publicUrl = replaceSensitiveData(PUBLIC_OVERPASS_URL) + "?data=" + overpassQuery;
+ String publicStats = retrieveOverpassStats(publicUrl, false);
+
+ // Get private overpass data if private overpass url was provided
+ String privateStats = null;
+ if (!replaceSensitiveData(PRIVATE_OVERPASS_URL).equals(PRIVATE_OVERPASS_URL)) {
+ String privateUrl = replaceSensitiveData(PRIVATE_OVERPASS_URL) + "?data=" + overpassQuery;
+ privateStats = retrieveOverpassStats(privateUrl, true);
+ }
JSONObject jobInfo = new JSONObject();
- jobInfo.put("overpassQuery", url);
- jobInfo.put("maxFeatureCount", MAX_OVERPASS_FEATURE_COUNT);
- jobInfo.put("railsCodename", railsCodename);
- jobInfo.put("overpassCodename", overpassCodename);
+ jobInfo.put("publicStats", publicStats);
+ jobInfo.put("privateStats", privateStats);
return Response.ok(jobInfo.toJSONString()).build();
}
@@ -665,25 +722,27 @@ public class GrailResource {
* a private OSM instance that has diverged from the public OSM
* with private changes.
*
- * @param bbox The bounding box
+ * @param reqParams Contains info such as bbox, layerName, and if the
+ * user provided one, a custom query for pulling overpass data
*
* @return Job ID
* used by the client for polling job status
* Internally, this provides the map dataset name suffix
*/
- @GET
+ @POST
@Path("/pullrailsporttodb")
@Produces(MediaType.APPLICATION_JSON)
public Response pullRailsPortToDb(@Context HttpServletRequest request,
- @QueryParam("bbox") String bbox,
- @QueryParam("name") String layerName) {
+ @QueryParam("folderId") Long folderId,
+ GrailParams reqParams) {
Users user = Users.fromRequest(request);
advancedUserCheck(user);
+ String bbox = reqParams.getBounds();
+ String layerName = reqParams.getInput1();
String jobId = UUID.randomUUID().toString().replace("-", "");
File workDir = new File(TEMP_OUTPUT_PATH, "grail_" + jobId);
- String folderName = "grail_" + bbox.replace(",", "_");
if (DbUtils.mapExists(layerName)) {
throw new BadRequestException("Record with name : " + layerName + " already exists. Please try a different name.");
@@ -698,11 +757,11 @@ public class GrailResource {
params.setWorkDir(workDir);
params.setOutput(layerName);
params.setBounds(bbox);
- params.setParentId(folderName);
+ params.setCustomQuery(reqParams.getCustomQuery());
List<Command> workflow;
try {
- workflow = setupRailsPull(jobId, params);
+ workflow = setupRailsPull(jobId, params, folderId);
}
catch(UnavailableException exc) {
return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(exc.getMessage()).build();
@@ -716,7 +775,7 @@ public class GrailResource {
return response;
}
- private List<Command> setupRailsPull(String jobId, GrailParams params) throws UnavailableException {
+ private List<Command> setupRailsPull(String jobId, GrailParams params, Long parentFolderId) throws UnavailableException {
List<Command> workflow = new LinkedList<>();
Users user = params.getUser();
@@ -726,14 +785,17 @@ public class GrailResource {
File referenceOSMFile = new File(params.getWorkDir(), REFERENCE +".osm");
if (referenceOSMFile.exists()) { referenceOSMFile.delete(); }
- params.setInput1(referenceOSMFile.getAbsolutePath());
+ GrailParams getRailsParams = new GrailParams(params);
+ getRailsParams.setOutput(referenceOSMFile.getAbsolutePath());
try {
- workflow.add(getRailsPortApiCommand(jobId, user, params.getBounds(), referenceOSMFile.getAbsolutePath()));
+ workflow.add(getRailsPortApiCommand(jobId, getRailsParams));
} catch (UnavailableException exc) {
throw new UnavailableException("The Rails port API is offline.");
}
+ params.setInput1(referenceOSMFile.getAbsolutePath());
+
// Write the data to the hoot db
ExternalCommand importRailsPort = grailCommandFactory.build(jobId, params, "info", PushToDbCommand.class, this.getClass());
workflow.add(importRailsPort);
@@ -745,11 +807,8 @@ public class GrailResource {
InternalCommand setMapTags = setMapTagsCommandFactory.build(tags, jobId);
workflow.add(setMapTags);
- // Create the folder if it doesn't exist
- Long folderId = DbUtils.createFolder(params.getParentId(), 0L, user.getId(), false);
-
// Move the data to the folder
- InternalCommand setFolder = updateParentCommandFactory.build(jobId, folderId, params.getOutput(), user, this.getClass());
+ InternalCommand setFolder = updateParentCommandFactory.build(jobId, parentFolderId, params.getOutput(), user, this.getClass());
workflow.add(setFolder);
return workflow;
@@ -826,4 +885,45 @@ public class GrailResource {
return params;
}
+ /**
+ *
+ * @param url
+ * @param usePrivateOverpass
+ * If true and the cert path is set then we know to use the cert for the overpass request
+ * If false then no cert will need to be used for the request
+ * @return
+ */
+ private static String retrieveOverpassStats(String url, boolean usePrivateOverpass) {
+ StringBuilder statsInfo = new StringBuilder();
+
+ try {
+ InputStream inputStream;
+
+ // if cert path is specified then we assume to use them for the request
+ // Both need to be true because in the case of using this function for public overpass we want it to skip immediately even if
+ // the cert path is specified.
+ if (usePrivateOverpass && !replaceSensitiveData(PRIVATE_OVERPASS_CERT_PATH).equals(PRIVATE_OVERPASS_CERT_PATH)) {
+ inputStream = PullApiCommand.getHttpResponseWithSSL(url);
+ } else {
+ URLConnection conn = new URL(url).openConnection();
+ inputStream = conn.getInputStream();
+ }
+
+ BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
+
+ String inputLine;
+ while ((inputLine = br.readLine()) != null) {
+ statsInfo.append(inputLine + "\n");
+ }
+
+ br.close();
+ }
+ catch (Exception exc) {
+ String msg = "Error retrieving overpass stats! Cause: " + exc.getMessage();
+ throw new WebApplicationException(exc, Response.status(Response.Status.NOT_FOUND).entity(msg).build());
+ }
+
+ return statsInfo.toString();
+ }
+
}