Merge pull request #82 from nasaharvest/stable-0.1.3

Stable 0.1.3
nasaharvest · Aug 21, 2022 · 2a2e832 · 2a2e832
2 parents 324aba9 + 9f5feef
commit 2a2e832
Show file tree

Hide file tree

Showing 8 changed files with 26 additions and 14 deletions.
diff --git a/buildings-example/openmapflow.yaml b/buildings-example/openmapflow.yaml
@@ -1,4 +1,4 @@
-version: 0.1.3rc1
+version: 0.1.3
 project: buildings-example
 description: OpenMapFlow buildings example
 gcloud:

diff --git a/crop-mask-example/openmapflow.yaml b/crop-mask-example/openmapflow.yaml
@@ -1,4 +1,4 @@
-version: 0.1.3rc1
+version: 0.1.3
 project: crop-mask-example
 description: OpenMapFlow crop mask example
 gcloud:

diff --git a/maize-example/openmapflow.yaml b/maize-example/openmapflow.yaml
@@ -1,4 +1,4 @@
-version: 0.1.3rc1
+version: 0.1.3
 project: maize-example
 description: OpenMapFlow maize example
 gcloud:

diff --git a/openmapflow/constants.py b/openmapflow/constants.py
@@ -13,7 +13,7 @@
 TEMPLATE_README = TEMPLATES_DIR / "README.md"
 TEMPLATE_DEPLOY_YML = TEMPLATES_DIR / "github-deploy.yaml"
 TEMPLATE_TEST_YML = TEMPLATES_DIR / "github-test.yaml"
-VERSION = "0.1.3rc1"
+VERSION = "0.1.3"
 
 # -------------- Dataframe column names --------------------------------------
 SOURCE = "source"

diff --git a/openmapflow/labeled_dataset.py b/openmapflow/labeled_dataset.py
@@ -107,13 +107,15 @@ def summary(self, df: pd.DataFrame) -> str:
             + "\n"
         )
 
-    def load_df(self, skip_to_np: bool = False) -> pd.DataFrame:
+    def load_df(
+        self, skip_to_np: bool = False, check_eo_data: bool = True
+    ) -> pd.DataFrame:
         """Load dataset (labels + earth observation data) as a DataFrame"""
         if not self.df_path.exists():
             print(self.create_dataset())
         df = pd.read_csv(self.df_path)
         df = df[clean_df_condition(df)].copy()
-        if df[EO_DATA].isnull().any():
+        if check_eo_data and df[EO_DATA].isnull().any():
             raise ValueError(
                 f"{self.dataset} has missing earth observation data, "
                 + "run openmapflow create-datasets"

diff --git a/openmapflow/labeled_dataset_custom.py b/openmapflow/labeled_dataset_custom.py
@@ -41,7 +41,7 @@
 from openmapflow.engineer import calculate_ndvi, fillna, load_tif, remove_bands
 from openmapflow.labeled_dataset import LabeledDataset, clean_df_condition
 from openmapflow.raw_labels import RawLabels
-from openmapflow.utils import memoized, tqdm
+from openmapflow.utils import tqdm
 
 temp_dir = tempfile.gettempdir()
 
@@ -72,7 +72,6 @@ def _distance_point_from_center(lat_idx: int, lon_idx: int, tif) -> int:
     return x_dist + y_dist
 
 
-@memoized
 def _generate_bbox_from_paths() -> Dict[Path, BBox]:
     cloud_eo_uris = get_cloud_tif_list(BucketNames.LABELED_EO)
     return {
@@ -369,12 +368,13 @@ def set_df(i, start, eo_paths, lon, lat, pbar):
                     df.at[i, EO_LON] = eo_lon
                     df.at[i, EO_FILE] = eo_file
                     df.at[i, EO_STATUS] = EO_STATUS_COMPLETE
+                return True
 
             with tqdm(
                 total=len(df_with_eo_files),
                 desc="Extracting matched earth observation data",
             ) as pbar:
-                np.vectorize(set_df)(
+                np.vectorize(set_df, otypes="b")(
                     i=df_with_eo_files.index,
                     start=df_with_eo_files[START],
                     eo_paths=df_with_eo_files[MATCHING_EO_FILES],

diff --git a/openmapflow/notebooks/new_data.ipynb b/openmapflow/notebooks/new_data.ipynb
@@ -24,6 +24,15 @@
         "If you don't already have one, obtain a Github Personal Access Token using the steps [here](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). Save this token somewhere private."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install \"ipywidgets>=7,<8\" -q # https://github.com/googlecolab/colabtools/issues/3020"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -81,7 +90,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install -r requirements.txt -q\n",
+        "!pip install openmapflow[data] dvc[gdrive] -q\n",
         "!pip install pyyaml==5.4.1 -q"
       ]
     },
@@ -280,8 +289,7 @@
       "outputs": [],
       "source": [
         "# Pushing to remote storage\n",
-        "!dvc commit {DataPaths.RAW_LABELS} -f -q\n",
-        "!dvc commit {DataPaths.DATASETS} -f -q\n",
+        "!dvc commit\n",
         "!dvc push"
       ]
     },
@@ -297,7 +305,7 @@
         "commit_message = input(\"Commit message: \")\n",
         "!git add .\n",
         "!git commit -m '{commit_message}'\n",
-        "!git push "
+        "!git push --set-upstream origin {branch_name}"
       ]
     },
     {
@@ -340,4 +348,4 @@
   },
   "nbformat": 4,
   "nbformat_minor": 0
-}
+}
diff --git a/openmapflow/templates/train.py b/openmapflow/templates/train.py
@@ -169,6 +169,8 @@ def forward(self, x):
 
         # ------------------------ Model saving --------------------------
         if lowest_validation_loss == val_loss:
+            # Some models in tsai need to be modified to be TorchScriptable
+            # https://github.com/timeseriesAI/tsai/issues/561
             sm = torch.jit.script(model)
             model_path = model_path_from_name(model_name=model_name)
             if model_path.exists():