Merge pull request #324 from CosmiQ/dev

Release v0.2.1 (merge to master)

CHANGELOG.md

@@ -23,6 +23,18 @@ When a new version of `solaris` is released, all of the changes in the Unrelease
 ### Deprecated
 ### Security
 
+---
+
+## Version 0.2.1
+
+### Changed
+- 20200103, nrweir: Updated version pins for proj6 compatibility, also relaxed version pins for many dependencies (#321)
+### Fixed
+- 20200103, nrweir: Fixed various places where CRS wasn't passed correctly from rasterio CRS object (#319, #322)
+- 20200103, nrweir: Fixed axis length check for axis ordering in sol.utils.raster.reorder_axes() (#318)
+
+---
+
 ## Version 0.2.0
 
 ### Added
@@ -44,6 +56,7 @@ When a new version of `solaris` is released, all of the changes in the Unrelease
 - 20191123, dphogan: Fixed issue in mask_to_poly_geojson() with empty GeoDataFrames.
 - 20191204, dphogan: Fixed issue with file output from footprint_mask() and contact_mask() (#301)
 - 20191212, jshermeyer: Fixed issue with vector tiling: could not load in list of sublists previously. Corrected comments for appropriate order as well. (#306)
+- 20191219: rbavery: In `solaris.utils.geo.split_geom`, tile bounds that fall within `aoi_boundary` but not `src_img` are not returned. `solaris.vector.mask.instance_mask` only rasterizes geojsons where `reference_im` has values (nodata pixels won't have corresponding labels) (#315)
 
 
 ---

docs/conf.py

@@ -24,8 +24,8 @@
 copyright = u'2018-{}, CosmiQ Works: an IQT Lab'.format(time.strftime("%Y"))
 
 # The full version, including alpha/beta/rc tags
-release = '0.2.0'
-version = '0.2.0'
+release = '0.2.1'
+version = '0.2.1'
 
 # -- General configuration ---------------------------------------------------
 

environment-gpu.yml

@@ -4,30 +4,30 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.6.7
-  - pip=19.0.3
-  - affine=2.3.0
+  - python>=3.6
+  - pip>=19.0.3
+  - affine>=2.3.0
   - albumentations=0.4.3
-  - fiona=1.8.13
-  - gdal=3.0.2
-  - geopandas=0.6.2
-  - matplotlib=3.1.2
-  - networkx=2.4
-  - numpy=1.17.3
-  - opencv=4.1.2
-  - pandas=0.25.3
-  - pyproj=2.4.2
+  - fiona>=1.7.13
+  - gdal>=3.0.2
+  - geopandas>=0.6.2
+  - matplotlib>=3.1.2
+  - networkx>=2.4
+  - numpy>=1.17.3
+  - opencv>=4.1
+  - pandas>=0.25.3
+  - pyproj>=2.1
   - pytorch=1.3.1
-  - torchvision=0.4.2
   - pyyaml=5.2
-  - rasterio=1.1.1
+  - rasterio>=1.0.23
   - requests=2.22.0
-  - rio-cogeo=1.1.6
-  - rtree=0.9.3
-  - scikit-image=0.16.2
-  - scipy=1.3.2
-  - shapely=1.6.4
-  - tqdm=4.40.0
-  - urllib3=1.25.7
+  - rio-cogeo>=1.1.6
+  - rtree>=0.9.3
+  - scikit-image>=0.16.2
+  - scipy>=1.3.2
+  - shapely>=1.6.4
+  - torchvision>=0.4.2
+  - tqdm>=4.40.0
+  - urllib3>=1.25.7
   - tensorflow-gpu=1.13.1
   - cuda92

environment.yml

@@ -4,29 +4,29 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.6.7
-  - pip=19.0.3
-  - affine=2.3.0
+  - python>=3.6
+  - pip>=19.0.3
+  - affine>=2.3.0
   - albumentations=0.4.3
-  - fiona=1.8.13
-  - gdal=3.0.2
-  - geopandas=0.6.2
-  - matplotlib=3.1.2
-  - networkx=2.4
-  - numpy=1.17.3
-  - opencv=4.1.2
-  - pandas=0.25.3
-  - pyproj=2.4.2
+  - fiona>=1.7.13
+  - gdal>=3.0.2
+  - geopandas>=0.6.2
+  - matplotlib>=3.1.2
+  - networkx>=2.4
+  - numpy>=1.17.3
+  - opencv>=4.1
+  - pandas>=0.25.3
+  - pyproj>=2.1
   - pytorch=1.3.1
   - pyyaml=5.2
-  - rasterio=1.1.1
+  - rasterio>=1.0.23
   - requests=2.22.0
-  - rio-cogeo=1.1.6
-  - rtree=0.9.3
-  - scikit-image=0.16.2
-  - scipy=1.3.2
-  - shapely=1.6.4
+  - rio-cogeo>=1.1.6
+  - rtree>=0.9.3
+  - scikit-image>=0.16.2
+  - scipy>=1.3.2
+  - shapely>=1.6.4
   - tensorflow=1.13.1
-  - torchvision=0.4.2
-  - tqdm=4.40.0
-  - urllib3=1.25.7
+  - torchvision>=0.4.2
+  - tqdm>=4.40.0
+  - urllib3>=1.25.7

requirements.txt

@@ -1,26 +1,26 @@
-python=3.6.7
-pip=19.0.3
-affine=2.3.0
+python>=3.6
+pip>=19.0.3
+affine>=2.3.0
 albumentations=0.4.3
-fiona=1.8.13
-gdal=3.0.2
-geopandas=0.6.2
-matplotlib=3.1.2
-networkx=2.4
-numpy=1.17.3
-opencv=4.1.2
-pandas=0.25.3
-pyproj=2.4.2
+fiona>=1.8.13
+gdal>=3.0.2
+geopandas>=0.6.2
+matplotlib>=3.1.2
+networkx>=2.4
+numpy>=1.17.3
+opencv=4.1.0.25
+pandas>=0.25.3
+pyproj>=2.1
 pytorch=1.3.1
 pyyaml=5.2
-rasterio=1.1.1
-requests=2.22.0
-rio-cogeo=1.1.6
-rtree=0.9.3
-scikit-image=0.16.2
-scipy=1.3.2
-shapely=1.6.4
+rasterio>=1.0.23
+requests>=2.22.0
+rio-cogeo>=1.1.6
+rtree>=0.9.3
+scikit-image>=0.16.2
+scipy>=1.3.2
+shapely>=1.6.4
 tensorflow=1.13.1
-torchvision=0.4.2
-tqdm=4.40.0
-urllib3=1.25.7
+torchvision>=0.4.2
+tqdm>=4.40.0
+urllib3>=1.25.7

setup.py

@@ -60,29 +60,31 @@ def check_output(cmd):
 if on_rtd:
     inst_reqs = ['sphinx_bootstrap_theme']
 else:
-    inst_reqs = ['shapely>=1.6.4',
-                 'fiona>=1.8.6',
-                 'pandas>=0.24.2',
-                 'geopandas>=0.4.1',
-                 'opencv-python==4.1.0.25',
-                 'numpy>=1.16.4',
-                 'tqdm>=4.32.2',
-                 'rtree>=0.8.3',
-                 'networkx>=2.3',
-                 'rasterio>=1.0.18',
-                 'rio-cogeo>=1.0.0',
-                 'scipy>=1.3.0',
-                 'urllib3==1.24.3',
-                 'scikit-image>=0.15.0',
-                 'tensorflow==1.13.1',
-                 'torch>=1.1.0',
-                 'matplotlib>=3.1.1',
-                 'affine>=2.2.2',
-                 'albumentations>=0.2.3',
+    inst_reqs = ['pip>=19.0.3',
+                 'affine>=2.3.0',
+                 'albumentations==0.4.3',
+                 'fiona>=1.8.13',
+                 'gdal>=3.0.2',
+                 'geopandas>=0.6.2',
+                 'matplotlib>=3.1.2',
+                 'networkx>=2.4',
+                 'numpy>=1.17.3',
+                 'opencv-python>=4.1.0.25',
+                 'pandas>=0.25.3',
+                 'pyproj>=2.1',
+                 'torch==1.3.1',
+                 'pyyaml==5.2',
+                 'rasterio>=1.0.23',
                  'requests>=2.22.0',
-                 # 'rio-tiler>=1.2.7',
-                 'pyyaml>=5.1',
-                 'torchvision>=0.3.0']
+                 'rio-cogeo>=1.1.6',
+                 'rtree>=0.9.3',
+                 'scikit-image>=0.16.2',
+                 'scipy>=1.3.2',
+                 'shapely>=1.6.4',
+                 'tensorflow==1.13.1',
+                 'torchvision>=0.4.2',
+                 'tqdm>=4.40.0',
+                 'urllib3>=1.25.7']
 
 
 extra_reqs = {

solaris/__init__.py

@@ -1,3 +1,3 @@
 from . import bin, data, eval, nets, raster, tile, utils, vector
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"

solaris/data/coco.py

@@ -125,7 +125,7 @@ def geojson2coco(image_src, label_src, output_path=None, image_ext='.tif',
     logger = logging.getLogger(__name__)
     logger.setLevel(_get_logging_level(int(verbose)))
     logger.debug('Preparing image filename: image ID dict.')
-
+    # pdb.set_trace()
     if isinstance(image_src, str):
         if image_src.endswith('json'):
             logger.debug('COCO json provided. Extracting fname:id dict.')
@@ -192,6 +192,8 @@ def geojson2coco(image_src, label_src, output_path=None, image_ext='.tif',
         logger.debug('Reading in {}'.format(gj))
         curr_gdf = gpd.read_file(gj)
         curr_gdf['label_fname'] = gj
+        curr_gdf['image_fname'] = ''
+        curr_gdf['image_id'] = np.nan
         if category_attribute is None:
             logger.debug('No category attribute provided. Creating a default '
                          '"other" category.')
@@ -202,12 +204,13 @@ def geojson2coco(image_src, label_src, output_path=None, image_ext='.tif',
         if do_matches:  # multiple images: multiple labels
             logger.debug('do_matches is True, finding matching image')
             logger.debug('Converting to pixel coordinates.')
-            curr_gdf = geojson_to_px_gdf(
-                curr_gdf,
-                im_path=match_df.loc[match_df['label_fname'] == gj,
-                                     'image_fname'].values[0])
-            curr_gdf['image_id'] = image_ref[match_df.loc[
-                match_df['label_fname'] == gj, 'image_fname'].values[0]]
+            if len(curr_gdf) > 0:  # if there are geoms, reproj to px coords
+                curr_gdf = geojson_to_px_gdf(
+                    curr_gdf,
+                    im_path=match_df.loc[match_df['label_fname'] == gj,
+                                         'image_fname'].values[0])
+                curr_gdf['image_id'] = image_ref[match_df.loc[
+                    match_df['label_fname'] == gj, 'image_fname'].values[0]]
         # handle case with multiple images, one big geojson
         elif len(image_ref) > 1 and len(label_list) == 1:
             logger.debug('do_matches is False. Many images:1 label detected.')

solaris/raster/image.py

@@ -2,7 +2,10 @@
 import rasterio
 from affine import Affine
 import numpy as np
+import logging
 from ..utils.raster import reorder_axes
+from ..utils.log import _get_logging_level
+import pdb
 
 
 def get_geo_transform(raster_src):
@@ -150,3 +153,100 @@ def stitch_images(im_arr, idx_refs=None, out_width=None,
     output_arr = output_arr.astype(im_arr.dtype)
 
     return output_arr
+
+
+def get_intensity_quantiles(dataset_dir, percentiles=[0, 100], ext="tif",
+                              recursive=False, channels=None, verbose=0):
+    """Get approximate dataset pixel intensity percentiles for normalization.
+
+    This function reads every image in a dataset directory and gets a rough
+    approximation of pixel intensity percentiles
+
+    Arguments
+    ---------
+    """
+    pass
+
+
+class ScaleFunction:
+    def __init__(self, compression_delta, **kwargs):
+        self.compression_delta = compression_delta
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+    def forward(self, quantile):
+        raise NotImplementedError
+
+    def inverse(self, k):
+        raise NotImplementedError
+
+
+class K1ScaleFunction(ScaleFunction):
+    """Calculate the k1 scale function for a quantile given a comp. factor."""
+
+    def __init__(self, compression_delta):
+        self.super().__init__(self, compression_delta)
+
+    def forward(self, quantile):
+        return (self.compression_delta/2*np.pi)*np.arcsin(2*quantile-1)
+
+    def inverse(self, k):
+        return np.sin((2*np.pi*k)/self.compression_delta)+1
+
+
+def get_tdigest(data_buffer, tdigest=None, scale_function=K1ScaleFunction,
+                compression_delta=0.01):
+    """Create a new t-digest or merge it with an existing digest.
+
+    This function is an implementation of Algorithm 1 from https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf
+
+    Arguments
+    ---------
+    data_buffer : :class:`numpy.ndarray`
+        An array of data to load into a tdigest. This will be flattened into
+        a vector.
+    tdigest : :class:`TDigest`, optional
+        An existing :class:`TDigest` object to merge with the new data.
+    """
+    pdb.set_trace()
+    buffer_centroids = data_buffer.flatten().astype(np.float32)
+    buffer_weights = np.ones(shape=buffer_centroids.shape, dtype=np.float32)
+    if tdigest:
+        buffer_centroids = np.concatenate((buffer_centroids,
+                                           tdigest.centroids))
+        buffer_weights = np.concatenate((buffer_weights, tdigest.weights))
+
+    # sort the data buffer on values (union with tdigest centroids if given)
+    sort_order = np.argsort(buffer_centroids)
+    buffer_centroids = buffer_centroids[sort_order]
+    buffer_weights = buffer_weights[sort_order]
+    S = buffer_weights.sum()
+    out_centroids = np.array([], dtype=np.float32)
+    out_weights = np.array([], dtype=np.float32)
+    q0 = 0.
+    q_limit = _get_q_limit(scale_function, q0, compression_delta)
+    sigma_centroid = buffer_centroids[0]
+    sigma_weight = buffer_weights[0]
+    for idx in range(1, len(buffer_centroids)):
+        q = q0 + (sigma_weight + buffer_weights[idx])/S
+        if q <= q_limit:
+            sigma_centroid = ((sigma_centroid*sigma_weight
+                               + buffer_centroids[idx]*buffer_weights[idx]) /
+                              (sigma_weight+buffer_weights[idx]))
+            sigma_weight = sigma_weight + buffer_weights[idx]
+        else:
+            np.append(out_centroids, sigma_centroid)
+            np.append(out_weights, sigma_weight)
+            q0 += sigma_weight/S
+            q_limit = _get_q_limit(scale_function, q0, compression_delta)
+            sigma_weight = buffer_weights[idx]
+            sigma_centroid = buffer_centroids[idx]
+    np.append(out_centroids, sigma_centroid)
+    np.append(out_weights, sigma_weight)
+
+    return out_centroids, out_weights
+
+
+def _get_q_limit(scale_function, q0, compression_delta):
+    return 1./(scale_function(scale_function(q0, compression_delta) + 1,
+                              compression_delta))