bug fixes for transit data models

All transit tests passing!
wsp-sag · Apr 16, 2024 · cb1d0d8 · cb1d0d8
1 parent 61748ee
commit cb1d0d8
Show file tree

Hide file tree

Showing 20 changed files with 129 additions and 923 deletions.
diff --git a/network_wrangler/models/_base/records.py b/network_wrangler/models/_base/records.py
@@ -23,12 +23,18 @@ class RecordModel(BaseModel):
         require_one_of (ClassVar[OneOf]): Class variable specifying fields that require exactly
             one of them to be present.
     """
-
-    model_config = ConfigDict(protected_namespaces=(), validate_assignment=True)
+    model_config = ConfigDict(
+        extra="forbid",
+        validate_assignment=True,
+        exclude_none=True,
+        protected_namespaces=()
+    )
     require_any_of: ClassVar[AnyOf] = []
     require_one_of: ClassVar[OneOf] = []
     _examples: ClassVar = []
 
+
+
     @staticmethod
     def _check_field_exists(field: Union[str, List[str]], values: Dict[str, Any]):
         if isinstance(field, list):

diff --git a/network_wrangler/projects/models.py b/network_wrangler/projects/models.py
@@ -12,14 +12,18 @@
 
 class SelectRouteProperties(RecordModel):
     """Selection properties for transit routes."""
-
-    model_config = ConfigDict(extra="allow")
-
     route_short_name: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)]
     route_long_name: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)]
     agency_id: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)]
     route_type: Annotated[Optional[List[int]], Field(None, min_length=1)]
 
+    model_config = ConfigDict(
+        extra="allow",
+        validate_assignment=True,
+        exclude_none=True,
+        protected_namespaces=()
+    )
+
 
 SelectionRequire = Union[Literal["any"], Literal["all"]]
 
@@ -31,12 +35,18 @@ class SelectTransitNodes(RecordModel):
         # "stop_id", TODO Not implemented
         "model_node_id",
     ]
-    model_config = ConfigDict(extra="forbid")
 
     # stop_id: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)] TODO Not implemented
     model_node_id: Annotated[List[int], Field(min_length=1)]
     require: Optional[SelectionRequire] = "any"
 
+    model_config = ConfigDict(
+        extra="forbid",
+        validate_assignment=True,
+        exclude_none=True,
+        protected_namespaces=()
+    )
+
     _examples = [
         # {"stop_id": ["stop1", "stop2"], "require": "any"},  TODO Not implemented
         {"model_node_id": [1, 2], "require": "all"},
@@ -45,12 +55,16 @@ class SelectTransitNodes(RecordModel):
 
 class TransitABNodesModel(RecordModel):
     """Single transit link model."""
-
-    model_config = ConfigDict(extra="forbid")
-
     A: Optional[int] = None  # model_node_id
     B: Optional[int] = None  # model_node_id
 
+    model_config = ConfigDict(
+        extra="forbid",
+        validate_assignment=True,
+        exclude_none=True,
+        protected_namespaces=()
+    )
+
 
 class SelectTransitLinks(RecordModel):
     """Requirements for describing multiple transit links of a project card."""
@@ -60,12 +74,16 @@ class SelectTransitLinks(RecordModel):
         "model_link_id",
     ]
 
-    model_config = ConfigDict(extra="forbid")
-
     model_link_id: Annotated[Optional[List[int]], Field(min_length=1)] = None
     ab_nodes: Annotated[Optional[List[TransitABNodesModel]], Field(min_length=1)] = None
     require: Optional[SelectionRequire] = "any"
-
+
+    model_config = ConfigDict(
+        extra="forbid",
+        validate_assignment=True,
+        exclude_none=True,
+        protected_namespaces=()
+    )
     _examples = [
         {
             "ab_nodes": [{"A": "75520", "B": "66380"}, {"A": "66380", "B": "75520"}],
@@ -80,16 +98,20 @@ class SelectTransitLinks(RecordModel):
 
 class SelectTripProperties(RecordModel):
     """Selection properties for transit trips."""
-
-    model_config = ConfigDict(extra="allow")
-
     trip_id: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)]
     shape_id: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)]
     direction_id: Annotated[Optional[int], Field(None)]
     service_id: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)]
     route_id: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)]
     trip_short_name: Annotated[Optional[List[ForcedStr]], Field(None, min_length=1)]
 
+    model_config = ConfigDict(
+        extra="allow",
+        validate_assignment=True,
+        exclude_none=True,
+        protected_namespaces=()
+    )
+
 
 TimeString = Annotated[
     ForcedStr,
@@ -104,9 +126,15 @@ class SelectTripProperties(RecordModel):
 
 
 class SelectTransitTrips(RecordModel):
-    model_config = ConfigDict(extra="forbid")
     trip_properties: Optional[SelectTripProperties] = None
     route_properties: Optional[SelectRouteProperties] = None
     timespans: Annotated[Optional[List[Timespan]], Field(None, min_length=1)]
     nodes: Optional[SelectTransitNodes] = None
     links: Optional[SelectTransitLinks] = None
+
+    model_config = ConfigDict(
+        extra="forbid",
+        validate_assignment=True,
+        exclude_none=True,
+        protected_namespaces=()
+    )
diff --git a/network_wrangler/roadway/links.py b/network_wrangler/roadway/links.py
@@ -10,9 +10,6 @@
 import pandas as pd
 import pandera as pa
 
-from jsonschema import validate
-from jsonschema.exceptions import ValidationError
-from jsonschema.exceptions import SchemaError
 from pandera.typing import Series
 from pandera.typing.geopandas import GeoSeries
 from pandera import check_input, check_output
@@ -306,41 +303,6 @@ def shape_id_from_link_geometry(
     return shape_ids
 
 
-def validate_wrangler_links_file(
-    link_file, schema_location: Union[Path, str] = "roadway_network_link.json"
-):
-    """
-    Validate roadway network data link schema and output a boolean
-    """
-    schema_location = Path(schema_location)
-    if not schema_location.exists():
-        base_path = Path(__file__).resolve().parent / "schemas"
-        schema_location = base_path / schema_location
-
-    with open(schema_location) as schema_json_file:
-        schema = json.load(schema_json_file)
-
-    with open(link_file) as link_json_file:
-        json_data = json.load(link_json_file)
-
-    try:
-        validate(json_data, schema)
-        return True
-
-    except ValidationError as exc:
-        WranglerLogger.error("Failed Link schema validation: Validation Error")
-        WranglerLogger.error("Link File Loc:{}".format(link_file))
-        WranglerLogger.error("Path:{}".format(exc.path))
-        WranglerLogger.error(exc.message)
-
-    except SchemaError as exc:
-        WranglerLogger.error("Invalid Link Schema")
-        WranglerLogger.error("Link Schema Loc: {}".format(schema_location))
-        WranglerLogger.error(json.dumps(exc.message, indent=2))
-
-    return False
-
-
 @pd.api.extensions.register_dataframe_accessor("true_shape")
 class TrueShapeAccessor:
     def __init__(self, links_df: LinksSchema):

diff --git a/network_wrangler/roadway/nodes.py b/network_wrangler/roadway/nodes.py
@@ -10,9 +10,6 @@
 import pandas as pd
 import pandera as pa
 
-from jsonschema import validate
-from jsonschema.exceptions import ValidationError
-from jsonschema.exceptions import SchemaError
 from pandera import check_input, check_output, DataFrameModel
 from pandera.typing import Series
 from pandera.typing.geopandas import GeoSeries
@@ -97,7 +94,7 @@ def read_nodes(
     """Reads nodes and returns a geodataframe of nodes.
 
     Sets index to be a copy of the primary key.
-    Validates output dataframe using NodsSchema.
+    Validates output dataframe using NodesSchema.
 
     Args:
         filename (Path,str): file to read links in from.
@@ -236,42 +233,6 @@ def get_nodes(
         )
 
 
-def validate_wrangler_nodes_file(
-    node_file: str, schema_location: Union[str, Path] = "roadway_network_node.json"
-) -> bool:
-    """
-    Validate roadway network data node schema and output a boolean
-    """
-    schema_location = Path(schema_location)
-    schema_location = Path(schema_location)
-    if not schema_location.exists():
-        base_path = Path(__file__).resolve().parent / "schemas"
-        schema_location = base_path / schema_location
-
-    with open(schema_location) as schema_json_file:
-        schema = json.load(schema_json_file)
-
-    with open(node_file) as node_json_file:
-        json_data = json.load(node_json_file)
-
-    try:
-        validate(json_data, schema)
-        return True
-
-    except ValidationError as exc:
-        WranglerLogger.error("Failed Node schema validation: Validation Error")
-        WranglerLogger.error("Node File Loc:{}".format(node_file))
-        WranglerLogger.error("Node Schema Loc:{}".format(schema_location))
-        WranglerLogger.error(exc.message)
-
-    except SchemaError as exc:
-        WranglerLogger.error("Invalid Node Schema")
-        WranglerLogger.error("Node Schema Loc:{}".format(schema_location))
-        WranglerLogger.error(json.dumps(exc.message, indent=2))
-
-    return False
-
-
 @check_input(NodesSchema, inplace=True)
 def nodes_df_to_geojson(nodes_df: pd.DataFrame, properties: list):
     """

diff --git a/network_wrangler/roadway/shapes.py b/network_wrangler/roadway/shapes.py
@@ -8,10 +8,6 @@
 import geopandas as gpd
 import pandera as pa
 
-from jsonschema import validate
-from jsonschema.exceptions import ValidationError
-from jsonschema.exceptions import SchemaError
-
 from pandera import check_output, DataFrameModel
 from pandera.typing import Series
 from pandera.typing.geopandas import GeoSeries
@@ -126,39 +122,3 @@ def write_shapes(
 ) -> None:
     shapes_file = Path(out_dir) / f"{prefix}shape.{format}"
     write_table(shapes_df, shapes_file, overwrite=overwrite)
-
-
-def validate_wrangler_shapes_file(
-    shapes_file: str, schema_location: Union[Path, str] = "roadway_network_shape.json"
-) -> bool:
-    """
-    Validate roadway network data node schema and output a boolean
-    """
-    schema_location = Path(schema_location)
-    schema_location = Path(schema_location)
-    if not schema_location.exists():
-        base_path = Path(__file__).resolve().parent / "schemas"
-        schema_location = base_path / schema_location
-
-    with open(schema_location) as schema_json_file:
-        schema = json.load(schema_json_file)
-
-    with open(shapes_file) as node_json_file:
-        json_data = json.load(node_json_file)
-
-    try:
-        validate(json_data, schema)
-        return True
-
-    except ValidationError as exc:
-        WranglerLogger.error("Failed Shapes schema validation: Validation Error")
-        WranglerLogger.error("Shapes File Loc:{}".format(shapes_file))
-        WranglerLogger.error("Shapes Schema Loc:{}".format(schema_location))
-        WranglerLogger.error(exc.message)
-
-    except SchemaError as exc:
-        WranglerLogger.error("Invalid Shape Schema")
-        WranglerLogger.error("Shape Schema Loc:{}".format(schema_location))
-        WranglerLogger.error(json.dumps(exc.message, indent=2))
-
-    return False