PBF support for /expansion

CHANGELOG.md

@@ -114,6 +114,7 @@
    * ADDED: some missing documentation about request parameters [#4687](https://github.com/valhalla/valhalla/pull/4687)
    * ADDED: Consider more forward/backward tags for access restrictions and speeds [#4686](https://github.com/valhalla/valhalla/pull/4686)
    * CHANGED: change costmatrix max_distance threshold to a distance threshold instead of duration [#4672](https://github.com/valhalla/valhalla/pull/4672)
+   * ADDED: PBF support for expansion [#4614](https://github.com/valhalla/valhalla/pull/4614/) 
 
 ## Release Date: 2023-05-11 Valhalla 3.4.0
 * **Removed**

proto/CMakeLists.txt

@@ -12,7 +12,8 @@ set(protobuf_descriptors
   incidents.proto
   status.proto
   matrix.proto
-  isochrone.proto)
+  isochrone.proto
+  expansion.proto)
 
 if(ENABLE_DATA_TOOLS)
   # Only mjolnir needs the OSM PBF descriptors

proto/api.proto

@@ -9,6 +9,7 @@ import public "info.proto";       // statistics about the request, filled out by
 import public "status.proto";     // info for status endpoint
 import public "matrix.proto";     // the matrix results
 import public "isochrone.proto";  // the isochrone results
+import public "expansion.proto";  // the expansion results
 
 message Api {
   // this is the request to the api
@@ -20,11 +21,9 @@ message Api {
   Status status = 4;          // status
   Matrix matrix = 5;          // sources_to_targets
   Isochrone isochrone = 6;    // isochrone
-  //TODO: isochrone
-  //TODO: matrix
-  //TODO: locate
-  //TODO: height
-  //TODO: expansion
+  Expansion expansion = 7;    // expansion
+  //TODO: locate;
+  //TODO: height;
 
   // here we store a bit of info about what happened during request processing (stats/errors/warnings)
   Info info = 20;

proto/expansion.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+option optimize_for = LITE_RUNTIME;
+package valhalla;
+
+message Expansion {
+
+  message Geometry {
+    repeated sint32 coords = 1 [packed=true];
+  }
+
+  enum EdgeStatus {
+    connected = 0;
+    settled = 1;
+    reached = 2;
+  }
+
+  repeated uint32 costs = 1 [packed=true];
+  repeated uint32 durations = 2 [packed=true];
+  repeated uint32 distances = 3 [packed=true];
+  repeated EdgeStatus edge_status = 4;
+  repeated uint32 edge_id = 5 [packed=true];
+  repeated uint32 pred_edge_id = 6 [packed=true];
+
+  repeated Geometry geometries = 7;
+}

proto/options.proto

@@ -53,10 +53,10 @@ message PbfFieldSelector {
   bool status = 4;     // /status
   bool matrix = 5;     // sources_to_targets
   bool isochrone = 6;
+  bool expansion = 9;
   // TODO: enable these once we have objects for them
   // bool locate = 7;
   // bool height = 8;
-  // bool expansion = 9;
 }
 
 message AvoidEdge {

src/proto_conversions.cc

@@ -436,4 +436,14 @@ travel_mode_type(const valhalla::DirectionsLeg_Maneuver& maneuver) {
       throw std::logic_error("Unknown travel mode");
   }
 }
+
+const std::string& Expansion_EdgeStatus_Enum_Name(const Expansion_EdgeStatus status) {
+  static const std::unordered_map<int, std::string> statuses{
+      {Expansion_EdgeStatus_reached, "r"},
+      {Expansion_EdgeStatus_settled, "s"},
+      {Expansion_EdgeStatus_connected, "c"},
+  };
+  auto i = statuses.find(status);
+  return i == statuses.cend() ? empty_str : i->second;
+}
 } // namespace valhalla

src/thor/bidirectional_astar.cc

@@ -361,7 +361,7 @@ inline bool BidirectionalAStar::ExpandInner(baldr::GraphReader& graphreader,
             ? GraphId{}
             : (FORWARD ? edgelabels_forward_ : edgelabels_reverse_)[pred.predecessor()].edgeid();
     expansion_callback_(graphreader, FORWARD ? meta.edge_id : opp_edge_id, prev_pred,
-                        "bidirectional_astar", "r", newcost.secs,
+                        "bidirectional_astar", Expansion_EdgeStatus_reached, newcost.secs,
                         pred.path_distance() + meta.edge->length(), newcost.cost);
   }
 
@@ -707,8 +707,9 @@ BidirectionalAStar::GetBestPath(valhalla::Location& origin,
         const auto prev_pred = fwd_pred.predecessor() == kInvalidLabel
                                    ? GraphId{}
                                    : edgelabels_forward_[fwd_pred.predecessor()].edgeid();
-        expansion_callback_(graphreader, fwd_pred.edgeid(), prev_pred, "bidirectional_astar", "s",
-                            fwd_pred.cost().secs, fwd_pred.path_distance(), fwd_pred.cost().cost);
+        expansion_callback_(graphreader, fwd_pred.edgeid(), prev_pred, "bidirectional_astar",
+                            Expansion_EdgeStatus_settled, fwd_pred.cost().secs,
+                            fwd_pred.path_distance(), fwd_pred.cost().cost);
       }
 
       // Prune path if predecessor is not a through edge or if the maximum
@@ -754,8 +755,9 @@ BidirectionalAStar::GetBestPath(valhalla::Location& origin,
         const auto prev_pred = rev_pred.predecessor() == kInvalidLabel
                                    ? GraphId{}
                                    : edgelabels_reverse_[rev_pred.predecessor()].edgeid();
-        expansion_callback_(graphreader, rev_pred.opp_edgeid(), prev_pred, "bidirectional_astar", "s",
-                            rev_pred.cost().secs, rev_pred.path_distance(), rev_pred.cost().cost);
+        expansion_callback_(graphreader, rev_pred.opp_edgeid(), prev_pred, "bidirectional_astar",
+                            Expansion_EdgeStatus_settled, rev_pred.cost().secs,
+                            rev_pred.path_distance(), rev_pred.cost().cost);
       }
 
       // Prune path if predecessor is not a through edge
@@ -869,8 +871,9 @@ bool BidirectionalAStar::SetForwardConnection(GraphReader& graphreader, const BD
     const auto prev_pred = pred.predecessor() == kInvalidLabel
                                ? GraphId{}
                                : edgelabels_forward_[pred.predecessor()].edgeid();
-    expansion_callback_(graphreader, pred.edgeid(), prev_pred, "bidirectional_astar", "c",
-                        pred.cost().secs, pred.path_distance(), pred.cost().cost);
+    expansion_callback_(graphreader, pred.edgeid(), prev_pred, "bidirectional_astar",
+                        Expansion_EdgeStatus_connected, pred.cost().secs, pred.path_distance(),
+                        pred.cost().cost);
   }
 
   return true;
@@ -941,8 +944,9 @@ bool BidirectionalAStar::SetReverseConnection(GraphReader& graphreader, const BD
     const auto prev_pred = fwd_pred.predecessor() == kInvalidLabel
                                ? GraphId{}
                                : edgelabels_forward_[fwd_pred.predecessor()].edgeid();
-    expansion_callback_(graphreader, fwd_edge_id, prev_pred, "bidirectional_astar", "c",
-                        fwd_pred.cost().secs, fwd_pred.path_distance(), fwd_pred.cost().cost);
+    expansion_callback_(graphreader, fwd_edge_id, prev_pred, "bidirectional_astar",
+                        Expansion_EdgeStatus_connected, fwd_pred.cost().secs,
+                        fwd_pred.path_distance(), fwd_pred.cost().cost);
   }
 
   return true;
@@ -1025,7 +1029,8 @@ void BidirectionalAStar::SetOrigin(GraphReader& graphreader,
 
     // setting this edge as reached
     if (expansion_callback_) {
-      expansion_callback_(graphreader, edgeid, GraphId{}, "bidirectional_astar", "r", cost.secs,
+      expansion_callback_(graphreader, edgeid, GraphId{}, "bidirectional_astar",
+                          Expansion_EdgeStatus_reached, cost.secs,
                           static_cast<uint32_t>(edge.distance() + 0.5), cost.cost);
     }
 
@@ -1122,7 +1127,8 @@ void BidirectionalAStar::SetDestination(GraphReader& graphreader,
 
     // setting this edge as reached, sending the opposing because this is the reverse tree
     if (expansion_callback_) {
-      expansion_callback_(graphreader, edgeid, GraphId{}, "bidirectional_astar", "r", cost.secs,
+      expansion_callback_(graphreader, edgeid, GraphId{}, "bidirectional_astar",
+                          Expansion_EdgeStatus_reached, cost.secs,
                           static_cast<uint32_t>(edge.distance() + 0.5), cost.cost);
     }
 

src/thor/costmatrix.cc

@@ -569,8 +569,8 @@ bool CostMatrix::ExpandInner(baldr::GraphReader& graphreader,
 
   // setting this edge as reached
   if (expansion_callback_) {
-    expansion_callback_(graphreader, meta.edge_id, pred.edgeid(), "costmatrix", "r", newcost.secs,
-                        pred_dist, newcost.cost);
+    expansion_callback_(graphreader, meta.edge_id, pred.edgeid(), "costmatrix",
+                        Expansion_EdgeStatus_reached, newcost.secs, pred_dist, newcost.cost);
   }
 
   return !(pred.not_thru_pruning() && meta.edge->not_thru());
@@ -613,8 +613,9 @@ bool CostMatrix::Expand(const uint32_t index,
   if (expansion_callback_) {
     auto prev_pred =
         pred.predecessor() == kInvalidLabel ? GraphId{} : edgelabels[pred.predecessor()].edgeid();
-    expansion_callback_(graphreader, pred.edgeid(), prev_pred, "costmatrix", "s", pred.cost().secs,
-                        pred.path_distance(), pred.cost().cost);
+    expansion_callback_(graphreader, pred.edgeid(), prev_pred, "costmatrix",
+                        Expansion_EdgeStatus_settled, pred.cost().secs, pred.path_distance(),
+                        pred.cost().cost);
   }
 
   if (FORWARD) {
@@ -847,8 +848,9 @@ void CostMatrix::CheckForwardConnections(const uint32_t source,
       auto prev_pred = fwd_pred.predecessor() == kInvalidLabel
                            ? GraphId{}
                            : edgelabel_[MATRIX_FORW][source][fwd_pred.predecessor()].edgeid();
-      expansion_callback_(graphreader, fwd_pred.edgeid(), prev_pred, "costmatrix", "c",
-                          fwd_pred.cost().secs, fwd_pred.path_distance(), fwd_pred.cost().cost);
+      expansion_callback_(graphreader, fwd_pred.edgeid(), prev_pred, "costmatrix",
+                          Expansion_EdgeStatus_connected, fwd_pred.cost().secs,
+                          fwd_pred.path_distance(), fwd_pred.cost().cost);
     }
   }
 
@@ -954,8 +956,9 @@ void CostMatrix::CheckReverseConnections(const uint32_t target,
         auto prev_pred = rev_pred.predecessor() == kInvalidLabel
                              ? GraphId{}
                              : edgelabel_[MATRIX_REV][source][rev_pred.predecessor()].edgeid();
-        expansion_callback_(graphreader, rev_pred.edgeid(), prev_pred, "costmatrix", "c",
-                            rev_pred.cost().secs, rev_pred.path_distance(), rev_pred.cost().cost);
+        expansion_callback_(graphreader, rev_pred.edgeid(), prev_pred, "costmatrix",
+                            Expansion_EdgeStatus_connected, rev_pred.cost().secs,
+                            rev_pred.path_distance(), rev_pred.cost().cost);
       }
     }
   }

src/thor/dijkstras.cc

@@ -386,8 +386,9 @@ void Dijkstras::Compute(google::protobuf::RepeatedPtrField<valhalla::Location>&
       const auto prev_pred = pred.predecessor() == kInvalidLabel
                                  ? GraphId{}
                                  : bdedgelabels_[pred.predecessor()].edgeid();
-      expansion_callback_(graphreader, pred.edgeid(), prev_pred, "dijkstras", "s", pred.cost().secs,
-                          pred.path_distance(), pred.cost().cost);
+      expansion_callback_(graphreader, pred.edgeid(), prev_pred, "dijkstras",
+                          Expansion_EdgeStatus_settled, pred.cost().secs, pred.path_distance(),
+                          pred.cost().cost);
     }
   }
 }

src/thor/expansion_action.cc

@@ -6,9 +6,53 @@
 #include "midgard/logging.h"
 #include "midgard/polyline2.h"
 #include "midgard/util.h"
+#include "tyr/serializers.h"
 
 using namespace rapidjson;
 using namespace valhalla::midgard;
+using namespace valhalla::tyr;
+
+namespace {
+
+using namespace valhalla;
+
+void writeExpansionProgress(Expansion* expansion,
+                            const baldr::GraphId& edgeid,
+                            const baldr::GraphId& prev_edgeid,
+                            const std::vector<midgard::PointLL>& shape,
+                            const std::unordered_set<Options::ExpansionProperties>& exp_props,
+                            const Expansion_EdgeStatus& status,
+                            const float& duration,
+                            const uint32_t& distance,
+                            const float& cost) {
+
+  auto* geom = expansion->add_geometries();
+  // make the geom
+  for (const auto& p : shape) {
+    geom->add_coords(round(p.lng() * 1e6));
+    geom->add_coords(round(p.lat() * 1e6));
+  }
+
+  // no properties asked for, don't collect any
+  if (!exp_props.size()) {
+    return;
+  }
+
+  // make the properties
+  if (exp_props.count(Options_ExpansionProperties_duration))
+    expansion->add_durations(static_cast<uint32_t>(duration));
+  if (exp_props.count(Options_ExpansionProperties_distance))
+    expansion->add_distances(static_cast<uint32_t>(distance));
+  if (exp_props.count(Options_ExpansionProperties_cost))
+    expansion->add_costs(static_cast<uint32_t>(cost));
+  if (exp_props.count(Options_ExpansionProperties_edge_status))
+    expansion->add_edge_status(status);
+  if (exp_props.count(Options_ExpansionProperties_edge_id))
+    expansion->add_edge_id(static_cast<uint32_t>(edgeid));
+  if (exp_props.count(Options_ExpansionProperties_pred_edge_id))
+    expansion->add_pred_edge_id(static_cast<uint32_t>(prev_edgeid));
+}
+} // namespace
 
 namespace valhalla {
 namespace thor {
@@ -30,21 +74,16 @@ std::string thor_worker_t::expansion(Api& request) {
     exp_props.insert(static_cast<Options_ExpansionProperties>(prop));
   }
 
-  // default the expansion geojson so its easy to add to as we go
-  writer_wrapper_t writer(1024 * 1024);
-  writer.start_object();
-  writer("type", "FeatureCollection");
-  writer.start_array("features");
-  writer.set_precision(6);
-
+  auto* expansion = request.mutable_expansion();
   // a lambda that the path algorithm can call to add stuff to the dom
   // route and isochrone produce different GeoJSON properties
   std::string algo = "";
-  auto track_expansion = [&writer, &opp_edges, &gen_factor, &skip_opps, &exp_props,
+  auto track_expansion = [&expansion, &opp_edges, &gen_factor, &skip_opps, &exp_props,
                           &algo](baldr::GraphReader& reader, baldr::GraphId edgeid,
                                  baldr::GraphId prev_edgeid, const char* algorithm = nullptr,
-                                 std::string status = nullptr, const float duration = 0.f,
-                                 const uint32_t distance = 0, const float cost = 0.f) {
+                                 const Expansion_EdgeStatus status = Expansion_EdgeStatus_reached,
+                                 const float duration = 0.f, const uint32_t distance = 0,
+                                 const float cost = 0.f) {
     algo = algorithm;
 
     auto tile = reader.GetGraphTile(edgeid);
@@ -73,51 +112,8 @@ std::string thor_worker_t::expansion(Api& request) {
       std::reverse(shape.begin(), shape.end());
     Polyline2<PointLL>::Generalize(shape, gen_factor, {}, false);
 
-    writer.start_object(); // feature object
-    writer("type", "Feature");
-
-    writer.start_object("geometry");
-    writer("type", "LineString");
-    writer.start_array("coordinates");
-
-    // make the geom
-    for (const auto& p : shape) {
-      writer.start_array();
-      writer(p.lng());
-      writer(p.lat());
-      writer.end_array();
-    }
-
-    writer.end_array();  // coordinates
-    writer.end_object(); // geometry
-
-    writer.start_object("properties");
-    // no properties asked for, don't collect any
-    if (!exp_props.size()) {
-      writer.end_object(); // properties
-      writer.end_object(); // feature
-      return;
-    }
-
-    // make the properties
-    if (exp_props.count(Options_ExpansionProperties_duration)) {
-      writer("duration", static_cast<uint64_t>(duration));
-    }
-    if (exp_props.count(Options_ExpansionProperties_distance)) {
-      writer("distance", static_cast<uint64_t>(distance));
-    }
-    if (exp_props.count(Options_ExpansionProperties_cost)) {
-      writer("cost", static_cast<uint64_t>(cost));
-    }
-    if (exp_props.count(Options_ExpansionProperties_edge_status))
-      writer("edge_status", status);
-    if (exp_props.count(Options_ExpansionProperties_edge_id))
-      writer("edge_id", static_cast<uint64_t>(edgeid));
-    if (exp_props.count(Options_ExpansionProperties_pred_edge_id))
-      writer("pred_edge_id", static_cast<uint64_t>(prev_edgeid));
-
-    writer.end_object(); // properties
-    writer.end_object(); // feature
+    writeExpansionProgress(expansion, edgeid, prev_edgeid, shape, exp_props, status, duration,
+                           distance, cost);
   };
 
   // tell all the algorithms how to track expansion
@@ -150,13 +146,6 @@ std::string thor_worker_t::expansion(Api& request) {
     // anyway
   }
 
-  // close the GeoJSON
-  writer.end_array(); // features
-  writer.start_object("properties");
-  writer("algorithm", algo);
-  writer.end_object();
-  writer.end_object(); // object
-
   // tell all the algorithms to stop tracking the expansion
   for (auto* alg : std::vector<PathAlgorithm*>{&multi_modal_astar, &timedep_forward, &timedep_reverse,
                                                &bidir_astar, &bss_astar}) {
@@ -166,7 +155,7 @@ std::string thor_worker_t::expansion(Api& request) {
   isochrone_gen.SetInnerExpansionCallback(nullptr);
 
   // serialize it
-  return writer.get_buffer();
+  return tyr::serializeExpansion(request, algo);
 }
 
 } // namespace thor

src/tyr/CMakeLists.txt

@@ -8,7 +8,8 @@ set(sources
   route_serializer_osrm.cc
   route_summary_cache.cc
   serializers.cc
-  transit_available_serializer.cc)
+  transit_available_serializer.cc
+  expansion_serializer.cc)
 
 set(sources_with_warnings
   locate_serializer.cc