Get elevation data from a pure Rust crate that only handles some

GeoTIFFs. #82

apps/game/src/edit/mod.rs

@@ -18,7 +18,7 @@ pub use self::roads::RoadEditor;
 pub use self::routes::RouteEditor;
 pub use self::stop_signs::StopSignEditor;
 pub use self::traffic_signals::TrafficSignalEditor;
-pub use self::validate::{check_sidewalk_connectivity};
+pub use self::validate::check_sidewalk_connectivity;
 use crate::app::{App, Transition};
 use crate::common::{tool_panel, CommonState, Warping};
 use crate::debug::DebugMode;

cloud/worker_script.sh

@@ -32,22 +32,13 @@ mv abstreet-importer/dev/data/input data/input
 rm -rf abstreet-importer
 find data/input -name '*.gz' -print -exec gunzip '{}' ';'
 
-# Set up Docker, for the elevation data
-sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-echo \
-	  "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
-	    $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+# Install GDAL
 sudo apt-get update
-# Also sneak GDAL in there
-sudo apt-get install -y docker-ce docker-ce-cli containerd.io libgdal-dev
+sudo apt-get install -y libgdal-dev
 
 # Now do the big import!
 rm -fv data/input/us/seattle/raw_maps/huge_seattle.bin data/input/us/seattle/popdat.bin
-# Run this as root so Docker works. We could add the current user to the group,
-# but then we have to fiddle with the shell a weird way to pick up the change
-# immediately.
-sudo ./target/release/cli regenerate-everything --shard-num=$WORKER_NUM --num-shards=$NUM_WORKERS
+./target/release/cli regenerate-everything --shard-num=$WORKER_NUM --num-shards=$NUM_WORKERS
 
 # Upload the results
 ./target/release/updater incremental-upload --version=$EXPERIMENT_TAG

convert_osm/Cargo.toml

@@ -9,6 +9,7 @@ abstio = { path = "../abstio" }
 abstutil = { path = "../abstutil" }
 anyhow = { workspace = true }
 csv = { workspace = true }
+elevation = { git = "https://github.com/dabreegster/elevation" }
 fs-err = { workspace = true }
 geom = { workspace = true }
 kml = { path = "../kml" }

convert_osm/src/elevation.rs

@@ -1,148 +1,39 @@
-use std::io::{BufRead, BufReader, BufWriter, Write};
-use std::process::Command;
+use std::collections::HashMap;
+use std::io::BufReader;
 
 use anyhow::Result;
+use elevation::GeoTiffElevation;
 use fs_err::File;
-use osm2streets::IntersectionID;
-
 use geom::Distance;
-use raw_map::RawMap;
-
-pub fn add_data(map: &mut RawMap) -> Result<()> {
-    let input = format!("elevation_input_{}", map.name.as_filename());
-    let output = format!("elevation_output_{}", map.name.as_filename());
-
-    // TODO It'd be nice to include more timing breakdown here, but if we bail out early,
-    // it's tedious to call timer.stop().
-    let ids = generate_input(&input, map)?;
-
-    fs_err::create_dir_all(&output)?;
-    fs_err::create_dir_all(abstio::path_shared_input("elevation"))?;
-    let pwd = std::env::current_dir()?.display().to_string();
-    // Because elevation_lookups has so many dependencies, just depend on Docker.
-    // TODO This is only going to run on Linux, unless we can also build images for other OSes.
-    // TODO On Linux, data/input/shared/elevation files wind up being owned by root, due to how
-    // docker runs. For the moment, one workaround is to manually fix the owner afterwards:
-    // find data/ -user root -exec sudo chown $USER:$USER '{}' \;
-    let status = Command::new("docker")
-        .arg("run")
-        // Bind the input directory to the temporary place we just created
-        .arg("--mount")
-        .arg(format!(
-            "type=bind,source={pwd}/{input},target=/elevation/input,readonly"
-        ))
-        // We want to cache the elevation data sources in A/B Street's S3 bucket, so bind to
-        // our data/input/shared directory.
-        .arg("--mount")
-        .arg(format!(
-            "type=bind,source={},target=/elevation/data",
-            // Docker requires absolute paths
-            format!("{}/{}", pwd, abstio::path_shared_input("elevation"))
-        ))
-        .arg("--mount")
-        .arg(format!(
-            "type=bind,source={pwd}/{output},target=/elevation/output",
-        ))
-        .arg("-t")
-        // https://hub.docker.com/r/abstreet/elevation_lookups
-        .arg("abstreet/elevation_lookups")
-        .arg("python3")
-        .arg("main.py")
-        .arg("query")
-        // TODO How to tune this? Pretty machine dependant, and using ALL available cores may
-        // melt memory.
-        .arg("--n_threads=1")
-        .status()?;
-    if !status.success() {
-        bail!("Command failed: {}", status);
-    }
-
-    scrape_output(&output, map, ids)?;
-
-    // Clean up temporary files
-    fs_err::remove_file(format!("{input}/query"))?;
-    fs_err::remove_dir(input)?;
-    fs_err::remove_file(format!("{output}/query"))?;
-    fs_err::remove_dir(output)?;
 
-    Ok(())
-}
+use abstutil::Timer;
+use raw_map::RawMap;
 
-fn generate_input(input: &str, map: &RawMap) -> Result<Vec<(IntersectionID, IntersectionID)>> {
-    fs_err::create_dir_all(input)?;
-    let mut f = BufWriter::new(File::create(format!("{input}/query"))?);
-    let mut ids = Vec::new();
+pub fn add_data(map: &mut RawMap, path: &str, timer: &mut Timer) -> Result<()> {
+    // Get intersection points from road endpoints, to reduce the number of elevation lookups
+    let mut intersection_points = HashMap::new();
     for r in map.streets.roads.values() {
-        ids.push((r.src_i, r.dst_i));
-        // Sample points along the road. Smaller step size gives more detail, but is slower.
-        let mut pts = Vec::new();
-        for (pt, _) in r
-            .reference_line
-            .step_along(Distance::meters(5.0), Distance::ZERO)
-        {
-            pts.push(pt);
-        }
-        // Always ask for the intersection
-        if *pts.last().unwrap() != r.reference_line.last_pt() {
-            pts.push(r.reference_line.last_pt());
+        for (i, pt) in [
+            (r.src_i, r.reference_line.first_pt()),
+            (r.dst_i, r.reference_line.last_pt()),
+        ] {
+            intersection_points.insert(i, pt.to_gps(&map.streets.gps_bounds));
         }
-        for (idx, gps) in map
-            .streets
-            .gps_bounds
-            .convert_back(&pts)
-            .into_iter()
-            .enumerate()
-        {
-            write!(f, "{},{}", gps.x(), gps.y())?;
-            if idx != pts.len() - 1 {
-                write!(f, " ")?;
-            }
-        }
-        writeln!(f)?;
     }
-    Ok(ids)
-}
 
-fn scrape_output(
-    output: &str,
-    map: &mut RawMap,
-    ids: Vec<(IntersectionID, IntersectionID)>,
-) -> Result<()> {
-    let num_ids = ids.len();
-    let mut cnt = 0;
-    for (line, (src_i, dst_i)) in BufReader::new(File::open(format!("{output}/query"))?)
-        .lines()
-        .zip(ids)
-    {
-        cnt += 1;
-        let line = line?;
-        let mut values = Vec::new();
-        for x in line.split('\t') {
-            if let Ok(x) = x.parse::<f64>() {
-                if !x.is_finite() {
-                    // TODO Warn
-                    continue;
-                }
-                if x < 0.0 {
-                    // TODO Temporary
-                    continue;
-                }
-                values.push(Distance::meters(x));
-            } else {
-                // Blank lines mean the tool failed to figure out what happened
+    // TODO Download the file if needed?
+    let mut elevation = GeoTiffElevation::new(BufReader::new(File::open(path)?));
+
+    timer.start_iter("lookup elevation", intersection_points.len());
+    for (i, gps) in intersection_points {
+        timer.next();
+        if let Some(height) = elevation.get_height_for_lon_lat(gps.x() as f32, gps.y() as f32) {
+            if height < 0.0 {
                 continue;
             }
+            map.elevation_per_intersection
+                .insert(i, Distance::meters(height.into()));
         }
-        if values.len() != 4 {
-            error!("Elevation output line \"{}\" doesn't have 4 numbers", line);
-            continue;
-        }
-        // TODO Also put total_climb and total_descent on the roads
-        map.elevation_per_intersection.insert(src_i, values[0]);
-        map.elevation_per_intersection.insert(dst_i, values[1]);
-    }
-    if cnt != num_ids {
-        bail!("Output had {} lines, but we made {} queries", cnt, num_ids);
     }
 
     // Calculate the incline for each road here, before the road gets trimmed for intersection

convert_osm/src/lib.rs

@@ -1,6 +1,4 @@
 #[macro_use]
-extern crate anyhow;
-#[macro_use]
 extern crate log;
 
 use std::collections::{HashMap, HashSet};
@@ -29,7 +27,8 @@ pub struct Options {
     pub extra_buildings: Option<String>,
     /// Configure public transit using this URL to a static GTFS feed in .zip format.
     pub gtfs_url: Option<String>,
-    pub elevation: bool,
+    /// Path to a GeoTIFF file in EPSG:4326 to use for elevation data
+    pub elevation_geotiff: Option<String>,
     /// Only include crosswalks that match a `highway=crossing` OSM node.
     pub filter_crosswalks: bool,
 }
@@ -43,7 +42,7 @@ impl Options {
             private_offstreet_parking: PrivateOffstreetParking::FixedPerBldg(1),
             extra_buildings: None,
             gtfs_url: None,
-            elevation: false,
+            elevation_geotiff: None,
             filter_crosswalks: false,
         }
     }
@@ -138,9 +137,9 @@ pub fn convert(
         filter_crosswalks(&mut map, extract.crossing_nodes, pt_to_road, timer);
     }
 
-    if opts.elevation {
+    if let Some(ref path) = opts.elevation_geotiff {
         timer.start("add elevation data");
-        if let Err(err) = elevation::add_data(&mut map) {
+        if let Err(err) = elevation::add_data(&mut map, path, timer) {
             error!("No elevation data: {}", err);
         }
         timer.stop("add elevation data");

importer/src/map_config.rs

@@ -76,6 +76,12 @@ pub fn config_for_map(name: &MapName) -> convert_osm::Options {
             None
         },
         // We only have a few elevation sources working
-        elevation: name.city == CityName::new("us", "seattle") || name.city.country == "gb",
+        elevation_geotiff: if name.city == CityName::new("us", "seattle") {
+            Some("data/input/shared/elevation/king_county_2016_lidar.tif".to_string())
+        } else if name.city.country == "gb" {
+            Some("data/input/shared/elevation/UK-dem-50m-4326.tif".to_string())
+        } else {
+            None
+        },
     }
 }