Merge pull request #391 from CosmiQ/v0.3.0_merge

Release version 0.3.0

.gitignore

@@ -69,6 +69,9 @@ instance/
 # Scrapy stuff:
 .scrapy
 
+# Text editor backups:
+*~
+
 # Sphinx documentation
 docs/_build/
 

CHANGELOG.md

@@ -25,6 +25,19 @@ When a new version of `solaris` is released, all of the changes in the Unrelease
 
 ---
 
+## Version 0.3.0
+
+### Added
+- 20200701, dphogan: Added multimodal preprocessing subpackage (#360)
+- 20200716, dphogan: Added three-part tutorial for preprocessing subpackage
+### Fixed
+- 20200630, dphogan: Remove edge case numeric values from loss function tests (#358)
+- 20200706, dphogan: Added API reference entry for utils.data
+- 20200706, roshanr11: fixed `checkpoint_frequency` parameter in YAML files to work as intended
+- 20200706, roshanr11: tqdm progress bar fixed, follow-up on previous issue (#335)
+
+---
+
 ## Version 0.2.2
 
 ### Changed
@@ -39,7 +52,6 @@ When a new version of `solaris` is released, all of the changes in the Unrelease
 - 20200401, rbavery: fixed bug where aoi boundary was not intersected with src_img extent prior to tiling
 - 20200401, rbavery/nrweir: adapted `_check_crs` to convert `pyproj.CRS` or pass through `rasterio.crs.CRS` class when rasterio crs object is required (for example, reprojecting in the tilers)
 - 20200414, zaburo-ch: fixed `val_datagen` to point to the correct augmentation pipeline
----
 
 ---
 

docs/api/index.rst

@@ -11,6 +11,7 @@ Complete submodule documentation
 * `solaris.tile <tile.html>`_: Tiling functionality for imagery and vector labels
 * `solaris.raster <raster.html>`_: Raster (imagery) coordinate management and formatting
 * `solaris.vector <vector.html>`_: Vector (label) management and format interconversion
+* `solaris.preproc <preproc.html>`_: Preprocessing workflows for imagery and vector labels
 * `solaris.nets <nets.html>`_: Deep learning model ingestion, creation, training, and inference
 * `solaris.eval <eval.html>`_: Deep learning model performance evaluation
 * `solaris.utils <utils.html>`_: Utility functions for the above toolsets
@@ -25,6 +26,7 @@ Submodule summaries
    tile
    raster
    vector
+   preproc
    nets
    eval
    utils

docs/api/preproc.rst

@@ -0,0 +1,31 @@
+.. title:: solaris.preproc API reference
+
+``solaris.preproc`` API reference
+=================================
+
+.. contents::
+
+
+``solaris.preproc.pipesegment`` Preprocessing base class and control structures
+-------------------------------------------------------------------------------
+
+.. automodule:: solaris.preproc.pipesegment
+   :members:
+
+``solaris.preproc.image`` Preprocessing of geospatial imagery
+-------------------------------------------------------------
+
+.. automodule:: solaris.preproc.image
+   :members:
+
+``solaris.preproc.sar`` Preprocessing of SAR imagery
+----------------------------------------------------
+
+.. automodule:: solaris.preproc.sar
+   :members:
+
+``solaris.preproc.label`` Preprocessing of vector labels
+----------------------------------------------------------
+
+.. automodule:: solaris.preproc.label
+   :members:

docs/api/utils.rst

@@ -40,3 +40,9 @@
 
 .. automodule:: solaris.utils.raster
    :members:
+
+``solaris.utils.data`` Dataset CSV utilities
+--------------------------------------------
+
+.. automodule:: solaris.utils.data
+   :members:

docs/conf.py

@@ -14,6 +14,7 @@
 import sys
 sys.path.insert(0, os.path.abspath('..'))
 import sphinx_bootstrap_theme
+import numpy as np
 # -- Project information -----------------------------------------------------
 
 project = 'solaris'
@@ -24,8 +25,8 @@
 copyright = u'2018-{}, CosmiQ Works: an IQT Lab'.format(time.strftime("%Y"))
 
 # The full version, including alpha/beta/rc tags
-release = '0.2.2'
-version = '0.2.2'
+release = '0.3.0'
+version = '0.3.0'
 
 # -- General configuration ---------------------------------------------------
 
@@ -61,7 +62,8 @@
                         'numpy', 'gdal', 'tqdm', 'rtree', 'networkx',
                         'rasterio', 'scipy', 'skimage', 'tensorflow', 'torch',
                         'torchvision', 'yaml', 'affine', 'albumentations',
-                        'rio_tiler', 'PIL', 'matplotlib', 'rio_cogeo']
+                        'rio_tiler', 'PIL', 'matplotlib', 'rio_cogeo',
+                        'pyproj']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -187,7 +189,7 @@ def setup(app):
     "geopandas": ('http://geopandas.org/', None),
     "rtree": ('http://toblerity.org/rtree/', None),
     "shapely": ('https://shapely.readthedocs.io/en/stable/', None),
-    'numpy': ('http://docs.scipy.org/doc/numpy/', None),
+    'numpy': ('https://numpy.org/doc/stable/', None),
     'scipy': ('http://docs.scipy.org/doc/scipy/reference/', None),
     'PyTorch': ('http://pytorch.org/docs/master/', None)
     }

docs/tutorials/index.rst

@@ -71,6 +71,9 @@ the tutorials below.
 * `Converting pixel masks to vector labels <notebooks/api_mask_to_vector.ipynb>`_
 * `Scoring your model's performance with the solaris Python API <notebooks/api_evaluation_tutorial.ipynb>`_
 * `Creating COCO-formatted datasets <notebooks/api_coco_tutorial.ipynb>`_
+* `Preprocessing Part 1: Pipelines <notebooks/preprocessing_pipelines.ipynb>`_
+* `Preprocessing Part 2: Branching <notebooks/preprocessing_branching.ipynb>`_
+* `Preprocessing Part 3: SAR <notebooks/preprocessing_sar.ipynb>`_
 
 Reference
 =========

environment-gpu.yml

@@ -25,6 +25,7 @@ dependencies:
   - scikit-image>=0.16.2
   - scipy>=1.3.2
   - torchvision>=0.5.0
+  - cudatoolkit=9.2
   - tqdm>=4.40.0
   - urllib3>=1.25.7
   - tensorflow-gpu=1.13.1

environment.yml

@@ -28,6 +28,7 @@ dependencies:
   - shapely>=1.6.4
   - tensorflow=1.13.1
   - torchvision>=0.5.0
+  - cudatoolkit=9.2
   - tqdm>=4.40.0
   - urllib3>=1.25.7
   - pip:

solaris/__init__.py

@@ -1,3 +1,3 @@
-from . import bin, data, eval, nets, raster, tile, utils, vector
+from . import bin, data, eval, nets, preproc, raster, tile, utils, vector
 
-__version__ = "0.2.2"
+__version__ = "0.3.0"

solaris/bin/make_graphs.py

@@ -1,6 +1,6 @@
 import argparse
 import pandas as pd
-from tqdm import tqdm
+from tqdm.auto import tqdm
 from multiprocessing import Pool
 from ..vector.graph import geojson_to_graph
 from ..utils.cli import _func_wrapper

solaris/bin/make_masks.py

@@ -1,6 +1,6 @@
 import argparse
 import pandas as pd
-from tqdm import tqdm
+from tqdm.auto import tqdm
 from multiprocessing import Pool
 from ..vector.mask import df_to_px_mask
 from ..utils.cli import _func_wrapper

solaris/data/coco.py

@@ -4,7 +4,7 @@
 from ..vector.polygon import geojson_to_px_gdf, remove_multipolygons
 import numpy as np
 import rasterio
-from tqdm import tqdm
+from tqdm.auto import tqdm
 import json
 import os
 import pandas as pd
@@ -16,7 +16,7 @@ def geojson2coco(image_src, label_src, output_path=None, image_ext='.tif',
                  matching_re=None, category_attribute=None, score_attribute=None,
                  preset_categories=None, include_other=True, info_dict=None,
                  license_dict=None, recursive=False, override_crs=False,
-                 explode_all_multipolygons=False, remove_all_multipolygons=False, 
+                 explode_all_multipolygons=False, remove_all_multipolygons=False,
                  verbose=0):
     """Generate COCO-formatted labels from one or multiple geojsons and images.
 
@@ -111,11 +111,11 @@ def geojson2coco(image_src, label_src, output_path=None, image_ext='.tif',
         to ``True`` will induce solaris to descend into subdirectories to find
         files. By default, solaris does not traverse the directory tree.
     explode_all_multipolygons : bool, optional
-        Explode the multipolygons into individual geometries using sol.utils.geo.split_multi_geometries. 
-        Be sure to inspect which geometries are multigeometries, each individual geometries within these 
+        Explode the multipolygons into individual geometries using sol.utils.geo.split_multi_geometries.
+        Be sure to inspect which geometries are multigeometries, each individual geometries within these
         may represent artifacts rather than true labels.
     remove_all_multipolygons : bool, optional
-        Filters MultiPolygons and GeometryCollections out of each tile geodataframe. Alternatively you 
+        Filters MultiPolygons and GeometryCollections out of each tile geodataframe. Alternatively you
         can edit each polygon manually to be a polygon before converting to COCO format.
     verbose : int, optional
         Verbose text output. By default, none is provided; if ``True`` or
@@ -200,14 +200,14 @@ def geojson2coco(image_src, label_src, output_path=None, image_ext='.tif',
     for gj in tqdm(label_list):
         logger.debug('Reading in {}'.format(gj))
         curr_gdf = gpd.read_file(gj)
-        
+
         if remove_all_multipolygons is True and explode_all_multipolygons is True:
             raise ValueError("Only one of remove_all_multipolygons or explode_all_multipolygons can be set to True.")
         if remove_all_multipolygons is True and explode_all_multipolygons is False:
             curr_gdf = remove_multipolygons(curr_gdf)
         elif explode_all_multipolygons is True:
             curr_gdf = split_multi_geometries(curr_gdf)
-        
+
         curr_gdf['label_fname'] = gj
         curr_gdf['image_fname'] = ''
         curr_gdf['image_id'] = np.nan
@@ -422,7 +422,7 @@ def df_to_coco_annos(df, output_path=None, geom_col='geometry',
     def _row_to_coco(row, geom_col, category_id_col, image_id_col, score_col):
         "get a single annotation record from a row of temp_df."
         if score_col is None:
-            
+
             return {'id': row['annotation_id'],
                     'image_id': int(row[image_id_col]),
                     'category_id': int(row[category_id_col]),