Merge branch 'master' into cb-ignore-common-restrictions

.circleci/config.yml

@@ -35,7 +35,7 @@ jobs:
           cd build
           cmake .. -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=On -DCPACK_GENERATOR=DEB \
             -DENABLE_COMPILER_WARNINGS=On -DENABLE_WERROR=Off -DCMAKE_EXPORT_COMPILE_COMMANDS=On \
-            -DCMAKE_CXX_FLAGS="-fuse-ld=lld" -DLOGGING_LEVEL=INFO -DENABLE_PYTHON_BINDINGS=On
+            -DCMAKE_CXX_FLAGS="-fuse-ld=lld" -DLOGGING_LEVEL=INFO -DENABLE_PYTHON_BINDINGS=On -DENABLE_GDAL=On
       - run: python3 ./scripts/valhalla_build_config
       - run: make -C build -j8
       - run: make -C build utrecht_tiles
@@ -74,7 +74,7 @@ jobs:
           cd build
           cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=On -DENABLE_PYTHON_BINDINGS=On -DLOGGING_LEVEL=TRACE \
             -DCPACK_GENERATOR=DEB -DCPACK_PACKAGE_VERSION_SUFFIX="-0ubuntu1-$(lsb_release -sc)" -DENABLE_SANITIZERS=ON \
-            -DENABLE_SINGLE_FILES_WERROR=Off
+            -DENABLE_SINGLE_FILES_WERROR=Off -DENABLE_GDAL=On
       - run: make -C build -j8
       - run: make -C build utrecht_tiles
       - run: make -C build -j8 tests
@@ -106,7 +106,7 @@ jobs:
           mkdir build
           cd build
           cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=On -DENABLE_PYTHON_BINDINGS=On -DCPACK_GENERATOR=DEB \
-            -DCPACK_PACKAGE_VERSION_SUFFIX="-0ubuntu1-$(lsb_release -sc)" -DENABLE_SINGLE_FILES_WERROR=Off
+            -DCPACK_PACKAGE_VERSION_SUFFIX="-0ubuntu1-$(lsb_release -sc)" -DENABLE_SINGLE_FILES_WERROR=Off -DENABLE_GDAL=On
       - run: make -C build -j8
       - run: make -C build utrecht_tiles
       - run: make -C build -j8 tests

.github/workflows/mingw-build.yml

@@ -18,6 +18,7 @@ jobs:
             mingw64-boost \
             mingw64-protobuf \
             mingw64-geos \
+            mingw64-gdal \
             mingw64-python3 \
             protobuf-compiler
       - uses: actions/checkout@v2
@@ -39,6 +40,7 @@ jobs:
             -DENABLE_PYTHON_BINDINGS=ON \
             -DENABLE_CCACHE=OFF \
             -DENABLE_TESTS=OFF \
+            -DENABLE_GDAL=ON \
             -DLOGGING_LEVEL=DEBUG \
             -DBoost_PROGRAM_OPTIONS_LIBRARY=/usr/x86_64-w64-mingw32/sys-root/mingw/lib/libboost_program_options-mt-x64.dll.a \
             -DPYTHON_EXECUTABLE=/usr/x86_64-w64-mingw32/bin/python3 \

.github/workflows/osx.yml

@@ -40,7 +40,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=1 brew install autoconf automake protobuf cmake ccache libtool sqlite3 libspatialite luajit curl wget czmq lz4 spatialite-tools unzip boost
+          HOMEBREW_NO_AUTO_UPDATE=1 brew install autoconf automake protobuf cmake ccache libtool sqlite3 libspatialite luajit curl wget czmq lz4 spatialite-tools unzip boost gdal
           sudo python -m pip install --break-system-packages requests shapely
           git clone https://github.com/kevinkreiser/prime_server --recurse-submodules && cd prime_server && ./autogen.sh && ./configure && make -j$(sysctl -n hw.logicalcpu) && sudo make install
 
@@ -58,7 +58,7 @@ jobs:
             ccache-osx-
 
       - name: Configure CMake
-        run: cmake -B build -DENABLE_SINGLE_FILES_WERROR=OFF
+        run: cmake -B build -DENABLE_SINGLE_FILES_WERROR=OFF -DENABLE_GDAL=ON
 
       - name: Build Valhalla
         run: make -C build -j$(sysctl -n hw.logicalcpu)

.github/workflows/win.yml

@@ -106,7 +106,8 @@ jobs:
             -DENABLE_TESTS=OFF \
             -DENABLE_CCACHE=OFF \
             -DENABLE_SERVICES=OFF \
-            -DPREFER_EXTERNAL_DEPS=ON
+            -DPREFER_EXTERNAL_DEPS=ON \
+            -DENABLE_GDAL=ON
 
       - if: ${{ steps.vcpkg-restore.outputs.cache-hit != 'true' }}
         name: Save vcpkg packages (if cache miss)

CHANGELOG.md

@@ -36,6 +36,8 @@
    * FIXED: fix config generator with thor.costmatrix_allow_second_pass [#4567](https://github.com/valhalla/valhalla/pull/4567)
    * FIXED: infinite loop or other random corruption in isochrones when retrieving partial shape of an edge [#4547](https://github.com/valhalla/valhalla/pull/4547)
    * FIXED: Aggregation updates: update opposing local idx after aggregating the edges, added classification check for aggregation, and shortcut length changes [#4570](https://github.com/valhalla/valhalla/pull/4570)
+   * FIXED: Use helper function for only parsing out names from DirectedEdge when populating intersecting edges [#4604](https://github.com/valhalla/valhalla/pull/4604)  
+   * FIXED: Osmnode size reduction: Fixed excessive disk space for planet build [#4605](https://github.com/valhalla/valhalla/pull/4605)
 * **Enhancement**
    * UPDATED: French translations, thanks to @xlqian [#4159](https://github.com/valhalla/valhalla/pull/4159)
    * CHANGED: -j flag for multithreaded executables to override mjolnir.concurrency [#4168](https://github.com/valhalla/valhalla/pull/4168)
@@ -93,6 +95,7 @@
    * ADDED: Improved instructions for blind users [#3694](https://github.com/valhalla/valhalla/pull/3694)
    * FIXED: Fixed roundoff issue in Tiles Row and Col methods [#4585](https://github.com/valhalla/valhalla/pull/4585)
    * ADDED: isochrone proper polygon support & pbf output for isochrone [#4575](https://github.com/valhalla/valhalla/pull/4575)
+   * ADDED: return isotile grid as geotiff  [#4594](https://github.com/valhalla/valhalla/pull/4594)
    * ADDED: `ignore_non_vehicular_restrictions` parameter for truck costing [#4606](https://github.com/valhalla/valhalla/pull/4606)
 
 ## Release Date: 2023-05-11 Valhalla 3.4.0

CMakeLists.txt

@@ -7,7 +7,7 @@
 # This is secondary build configuration provided for convenient development
 # on Windows and using CMake-enabled IDEs.
 #
-cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 project(valhalla LANGUAGES CXX C)
 
 include(FindPkgConfig)
@@ -36,6 +36,7 @@ option(ENABLE_SINGLE_FILES_WERROR "Convert compiler warnings to errors for singl
 option(PREFER_EXTERNAL_DEPS "Whether to use internally vendored headers or find the equivalent external package" OFF)
 # useful to workaround issues likes this https://stackoverflow.com/questions/24078873/cmake-generated-xcode-project-wont-compile
 option(ENABLE_STATIC_LIBRARY_MODULES "If ON builds Valhalla modules as STATIC library targets" OFF)
+option(ENABLE_GDAL "Whether to include GDAL; currently only used for raster serialization of isotile grid" OFF)
 
 set(LOGGING_LEVEL "" CACHE STRING "Logging level, default is INFO")
 set_property(CACHE LOGGING_LEVEL PROPERTY STRINGS "NONE;ALL;ERROR;WARN;INFO;DEBUG;TRACE")
@@ -96,8 +97,6 @@ include(ValhallaSourceGroups)
 # We use pkg-config for (almost) all dependencies:
 # - CMake Find* modules are versioned with CMake, not the packages, see protobuf > 21.12
 # - it's more trivial to use custom installations via PKG_CONFIG_PATH env var
-find_package(PkgConfig REQUIRED)
-find_package(Threads REQUIRED)
 pkg_check_modules(ZLIB REQUIRED IMPORTED_TARGET zlib)
 pkg_check_modules(LZ4 REQUIRED IMPORTED_TARGET liblz4)
 
@@ -113,12 +112,19 @@ if (ENABLE_HTTP OR ENABLE_DATA_TOOLS)
   endif()
 endif()
 
+# prefer CONFIG mode over MODULE mode, which versions configuration on the package, not CMake
+# NOTE: this is only supported for cmake >= 3.15, but shouldn't be a problem in real life
+set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)
+
 # Boost has no .pc file..
 find_package(Boost 1.71 REQUIRED)
 add_definitions(-DBOOST_NO_CXX11_SCOPED_ENUMS)
 add_definitions(-DBOOST_ALLOW_DEPRECATED_HEADERS)
 add_definitions(-DBOOST_BIND_GLOBAL_PLACEHOLDERS)
 
+find_package(PkgConfig REQUIRED)
+find_package(Threads REQUIRED)
+
 # resolve vendored libraries
 set(date_include_dir ${VALHALLA_SOURCE_DIR}/third_party/date/include)
 set(rapidjson_include_dir ${CMAKE_SOURCE_DIR}/third_party/rapidjson/include)
@@ -167,14 +173,10 @@ endif()
 
 # Protobuf is non-trivial to include via pkg-config, pkg_check_modules has no way to check
 # for protoc location in a platform agnostic manner
-# prefer CONFIG mode over MODULE mode, which versions configuration on the package, not CMake
-# NOTE: this is only supported for cmake >= 3.15, but shouldn't be a problem in real life
-set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)
 # newer protobuf versions require a compat bool
 set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "")
 find_package(Protobuf REQUIRED)
 # and turn it off again
-set(CMAKE_FIND_PACKAGE_PREFER_CONFIG OFF)
 message(STATUS "Using protoc from ${Protobuf_PROTOC_EXECUTABLE}")
 message(STATUS "Using pbf headers from ${Protobuf_INCLUDE_DIRS}")
 message(STATUS "Using pbf libs from ${PROTOBUF_LIBRARIES}")
@@ -189,6 +191,16 @@ else()
   message(FATAL_ERROR "Required target protobuf::libprotobuf-lite or protobuf::libprotobuf is not defined")
 endif()
 
+# gdal
+set(gdal_target "")
+if (ENABLE_GDAL)
+  find_package(GDAL REQUIRED)
+  add_compile_definitions(ENABLE_GDAL)
+  set(gdal_target GDAL::GDAL)
+endif()
+
+set(CMAKE_FIND_PACKAGE_PREFER_CONFIG OFF)
+
 # libprime_server
 # see https://gitlab.kitware.com/cmake/cmake/-/issues/19467
 set(libprime_server_targets "")
@@ -203,6 +215,7 @@ if(ENABLE_SERVICES)
   endif()
 endif()
 
+
 ## Mjolnir and associated executables
 if(ENABLE_DATA_TOOLS)
   add_compile_definitions(DATA_TOOLS)

cmake/ValhallaPkgConfig.cmake

@@ -33,6 +33,9 @@ function(configure_valhalla_pc)
   if(ENABLE_SERVICES)
     list(APPEND REQUIRES libprime_server)
   endif()
+  if(ENABLE_GDAL)
+    list(APPEND REQUIRES gdal)
+  endif()
   if(WIN32 AND NOT MINGW)
     list(APPEND LIBS_PRIVATE -lole32 -lshell32)
   else()

docs/docs/api/isochrone/api-reference.md

@@ -51,9 +51,9 @@ The isochrone service uses the `auto`, `bicycle`, `pedestrian`, and `multimodal`
 
 ## Outputs of the Isochrone service
 
-In the service response, the isochrone contours are returned as [GeoJSON](http://geojson.org/), which can be integrated into mapping applications.
+In the service response, the isochrone contours are returned as [GeoJSON](http://geojson.org/), which can be integrated into mapping applications. Alternatively, the grid data that underlies these contours can be returned as a [GeoTIFF](https://www.ogc.org/standard/geotiff/). 
 
-The contours are calculated using rasters and are returned as either polygon or line features, depending on your input setting for the `polygons` parameter. If an isochrone request has been named using the optional `&id=` input, then the `id` is returned as a name property for the feature collection within the GeoJSON response. A `metric` attribute lets you know whether it's a `distance` or `time` contour. A warnings array may also be included. This array may contain warning objects informing about deprecated request parameters, clamped values etc. | 
+The isochrone service returns contours as GeoJSON line or polygon features for the requested intervals (depending on the value of the `polygons` request parameter). These contours are calculated using a two dimensional grid. If the `format` request parameter is set to `geotiff`, the underlying grid data is returned directly instead of the contours derived from it. It will return one band for each requested metric (i.e. one for `time` and one for `distance`). If an isochrone request has been named using the optional `&id=` input, then the `id` is returned as a name property for the feature collection within the GeoJSON response. A `metric` attribute lets you know whether it's a `distance` or `time` contour. A warnings array may also be included. This array may contain warning objects informing about deprecated request parameters, clamped values etc. | 
 
 See the [HTTP return codes](../turn-by-turn/api-reference.md#http-status-codes-and-conditions) for more on messages you might receive from the service.
 
@@ -63,6 +63,8 @@ Most JavaScript-based GeoJSON renderers, including [Leaflet](http://leafletjs.co
 
 When making a map, drawing the isochrone contours as lines is more straightforward than polygons, and, therefore, currently is the default and recommended method. When deciding between the output as lines and polygons, consider your use case and the additional styling considerations involved with polygons. For example, fills should be rendered as semi-transparent over the other map layers so they are visible, although you may have more flexibility when using a vector-based map. In addition, polygons from multiple contour levels do not have overlapping areas cut out or removed. In other words, the outer contours include the areas of any inner contours, causing the colors and transparencies to blend when multiple contour polygons are drawn at the same time.
 
+(TODO: write something about rendering the GeoTIFF output.)
+
 ## Future work on the isochrone service
 
 The Isochrone service is in active development. To report software issues or suggest enhancements, open an issue in the [Valhalla GitHub repository](https://github.com/valhalla/valhalla/issues).
@@ -75,7 +77,7 @@ Several other options are being considered as future service enhancements. These
 * ~~Removing self intersections from polygonal contours.~~
 * ~~Allowing multiple locations to compute the region reachable from any of the locations within a specified time.~~
 * ~~Generating contours with reverse access logic to see the region that can reach a specific location within the specified time.~~
-* Returning raster data for potential animation using OpenGL shaders. This also has analysis use for being able to query thousands of locations to determine the time to each location, including improvements with one-to-many requests to the Valhalla Time-Distance Matrix service.
+* ~~Returning raster data for potential animation using OpenGL shaders. This also has analysis use for being able to query thousands of locations to determine the time to each location, including improvements with one-to-many requests to the Valhalla Time-Distance Matrix service.~~
 
 ## Data credits
 

docs/docs/api/turn-by-turn/api-reference.md

@@ -624,4 +624,5 @@ The codes correspond to code returned from a particular [Valhalla project](https
 |**5xx** | **Tyr project codes** |
 |500 | Failed to parse intermediate request format |
 |501 | Failed to parse TripDirections |
+|504 | GeoTIFF serialization not supported by service |
 |599 | Unknown |

docs/docs/building.md

@@ -27,6 +27,7 @@ Important build options include:
 | `-DENABLE_ADDRESS_SANITIZER` (`ON` / `OFF`) | Build with address sanitizer (defaults to off).|
 | `-DENABLE_UNDEFINED_SANITIZER` (`ON` / `OFF`) | Build with undefined behavior sanitizer (defaults to off).|
 | `-DPREFER_SYSTEM_DEPS` (`ON` / `OFF`) | Whether to use internally vendored headers or find the equivalent external package (defaults to off).|
+| `-DENABLE_GDAL` (`ON` / `OFF`) | Whether to include GDAL as a dependency (used for GeoTIFF serialization of isochrone grid) (defaults to off).|
 
 ### Building with `vcpkg` - any platform
 
@@ -138,7 +139,7 @@ When importing `libvalhalla` as a dependency in a project, it's important to kno
 
 To resolve `libvalhalla`'s linker/library paths/options, we recommend to use `pkg-config` or `pkg_check_modules` (in CMake).
 
-Currently, `rapidjson`, `date` & `dirent` (Win only) headers are vendored in `third_party`. Consuming applications are encouraged to use `pkg-config` to resolve Valhalla and its dependencies which will automatically install those headers to `/path/to/include/valhalla/third_pary/{rapidjson, date, dirent.h}` and can be `#include`d appropriately.
+Currently, `rapidjson`, `date` & `dirent` (Win only) headers are vendored in `third_party`. Consuming applications are encouraged to use `pkg-config` to resolve Valhalla and its dependencies which will automatically install those headers to `/path/to/include/valhalla/third_party/{rapidjson, date, dirent.h}` and can be `#include`d appropriately.
 
 ## Running Valhalla server on Unix
 

docs/docs/thor/isochrones.md

@@ -27,7 +27,7 @@ The 2-D grid is used to find the isocrhone contours by using a well-known contou
 
 After forming sets of contour polygons, KEVIN -please write a paragraph or 2 to describe how the contours are formed and output!
 
-This 2-D grid of times can be useful for other purposes as well. It provides a very fast way to query a single location to see how long it takes to get there from the test location. Ultimately this could be a way to do very large one-to-many matrices. At this time we do not return the 2-D array of times, but this is a possibility in the future.
+This 2-D grid can be useful for other purposes as well. It provides a very fast way to query a single location to see how long it takes to get there from the test location. Ultimately this could be a way to do very large one-to-many matrices. 
 
 Where is it?
 ------------

proto/options.proto

@@ -341,6 +341,7 @@ message Options {
     gpx = 1;
     osrm = 2;
     pbf = 3;
+    geotiff = 4;
   }
 
   enum Action {

scripts/install-linux-deps.sh

@@ -22,6 +22,7 @@ env DEBIAN_FRONTEND=noninteractive sudo apt install --yes --quiet \
     libboost-all-dev \
     libcurl4-openssl-dev \
     libczmq-dev \
+    libgdal-dev \
     libgeos++-dev \
     libgeos-dev \
     libluajit-5.1-dev \

src/argparse_utils.h

@@ -21,6 +21,7 @@
  * @param config  The config which will be populated here
  * @param log     The logging config node's key. If empty, logging will not be configured.
  * @param use_threads Whether this program multi-threads
+ * @param extra_help Optional function pointer to print more stuff to the end of the help message.
  *
  * @returns true if the program should continue, false if we should EXIT_SUCCESS
  * @throws cxxopts::exceptions::exception Thrown if there's no valid configuration
@@ -30,9 +31,13 @@ bool parse_common_args(const std::string& program,
                        const cxxopts::ParseResult& result,
                        boost::property_tree::ptree& conf,
                        const std::string& log,
-                       const bool use_threads = false) {
+                       const bool use_threads = false,
+                       std::function<void()> extra_help = nullptr) {
   if (result.count("help")) {
     std::cout << opts.help() << "\n";
+    if (extra_help) {
+      extra_help();
+    };
     return false;
   }