reorganizing and adding docs

.gitignore

@@ -1,3 +1,6 @@
+# project specific
+notebooks/config.yaml
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.rst

@@ -1,5 +1,5 @@
-`concordia` - Harmonization, downscaling and gridding of energy data
-=================================
+`concordia` - Harmonization, downscaling and gridding of emissions data
+=======================================================================
 
 Install
 -------
@@ -26,7 +26,8 @@ You can then check to make sure your install is operating as expected
 
 Running
 -------
-The main entrypoint is `notebooks/workflow.ipynb`.
+The main entrypoint is `notebooks/workflow.ipynb`. See the README file in
+`notebooks`.
 
 License
 -------

notebooks/README.rst

@@ -0,0 +1,36 @@
+Workflows
+=========
+
+This folder provides workflows for generating harmonized, downscaled, and
+gridded data. All notebooks here are in the form of python files, which can be
+synced as jupyter notebooks with the
+[jupytext](https://jupytext.readthedocs.io/en/latest/paired-notebooks.html)
+tool, e.g.,
+
+.. code-block:: bash
+
+    jupytext --sync workflow.py # generates a workflow.ipynb file
+
+If you make a material change to `workflow.ipynb`, you can resync it manually if
+needed with
+
+.. code-block:: bash
+
+    jupytext --sync workflow.ipynb # updates the workflow.py file
+
+Configuration
+-------------
+
+We use a `config.yaml` file to define a variety of configuration parameters. To
+get started, copy the example file `example_config.yaml` and rename it to
+`config.yaml`.
+
+RESCUE
+======
+
+External data
+-------------
+
+External data needed is provided at [this link](TODO). It should be placed at
+the location in the `config.yaml`'s  `shared_data`, which is by default
+`concordia/data/rescue`.

notebooks/example_config.yaml

@@ -0,0 +1,42 @@
+out_path: "../results"
+data_path: "../data/<my project>" # this needs to be changed based on where you want to store your own data
+
+history_path: "$data_path/historical"
+scenario_path: "$data_path/scenarios"
+gridding_path: "$data_path/gridding_process_files"
+proxy_path: "$data_path/gridding_process_files/proxy_rasters"
+postprocess_path: "$data_path/postprocessing"
+
+encoding:
+  zlib: true
+  complevel: 2
+
+regionmappings:
+  MyModelName:
+    path: "$data_path/regionmapping_mymodelname.csv"
+    country_column: "CountryCode"
+    region_column: "RegionCode"
+    sep: ";"
+
+country_combinations:
+  sdn_ssd: ["ssd", "sdn"]
+  isr_pse: ["isr", "pse"]
+  srb_ksv: ["srb", "srb (kosovo)"]
+
+variable_template: "CEDS+|9+ Sectors|Emissions|{gas}|{sector}"
+base_year: 2020
+
+luc_sectors:
+  - "Agricultural Waste Burning"
+  - "Grassland Burning"
+  - "Forest Burning"
+  - "Peat Burning"
+  - "Agriculture"
+  - "Aggregate - Agriculture and LUC"
+  - "CDR Afforestation"
+
+ftp:
+  server: "127.0.0.1"
+  port: 21
+  user: anonymous
+  password: ""

notebooks/gridding_data/generate_ceds_proxy_netcdfs.py

@@ -38,7 +38,7 @@
 from concordia.settings import Settings
 
 
-settings = Settings.from_config("../config.yaml", version=None)
+settings = Settings.from_config("config.yaml", base_path="..", version=None)
 
 dim_order = ["gas", "sector", "level", "year", "month", "lat", "lon"]
 

notebooks/gridding_data/generate_non_ceds_proxy_netcdfs.py

@@ -14,6 +14,8 @@
 
 # %%
 import geoutils as gu
+import ptolemy as pt
+import pyogrio as pio
 import rasterio as rio
 import xarray as xr
 from ptolemy.raster import IndexRaster
@@ -24,13 +26,12 @@
 
 
 # %%
-settings = Settings.from_config("../config.yaml", version=None)
+settings = Settings.from_config("config.yaml", base_path="..", version=None)
 
 # %%
 dim_order = ["gas", "sector", "level", "year", "month", "lat", "lon"]
 
 # %%
-
 missing_countries = ReportMissingCountries(
     IndexRaster.from_netcdf(settings.gridding_path / "ssp_comb_indexraster.nc")
 )
@@ -95,7 +96,7 @@ def convert_mariteam_to_ceds(mari, gas):
 #
 # We provide proxies for several CDR technologies:
 #
-# 1. OAE CDR re-uses shipping CO2 emissions (to be updated and spread evenly into EEZ)
+# 1. OAE CDR uses a full ocean map (together with the country indexraster this results in OAE being applied equally to within country's EEZ)
 # 2. DACCS CDR incorporates renewable potentials and CO2 storage potentials
 # 3. Industry CDR uses the composition of renewables, CO2 storage and industry co2 emissions
 #
@@ -107,30 +108,41 @@ def convert_mariteam_to_ceds(mari, gas):
 ).emissions
 ind_co2
 
-# %% [markdown]
-# # OAE CDR and emissions
-#
+# %%
+ind_co2_dimensions = xr.ones_like(ind_co2.drop_vars("sector"))
+ind_co2_seasonality = ind_co2.sum(["gas", "year"])
+ind_co2_seasonality /= ind_co2_seasonality.sum(["lat", "lon"]).mean("month")
 
 # %% [markdown]
+# ## OAE CDR and emissions
 #
 
 # %% [markdown]
-# Use shipping CO2 for OAE CDR emissions
-#
+# Rasterize natural earth ocean shape to proxy grids.
 
 # %%
+rasterize = pt.Rasterize(
+    shape=(ind_co2.sizes["lat"], ind_co2.sizes["lon"]),
+    coords={"lat": ind_co2.coords["lat"], "lon": ind_co2.coords["lon"]},
+)
+rasterize.read_shpf(
+    pio.read_dataframe(
+        settings.gridding_path / "non_ceds_input" / "ne_10m_ocean"
+    ).reset_index(),
+    idxkey="index",
+)
 oae_cdr = (
-    (xr.open_dataset(settings.proxy_path / "shipping_CO2.nc"))
-    .emissions.sel(sector="SHP")
-    .assign_coords(sector="OAE_CDR")
+    rasterize.rasterize(strategy="weighted", normalize_weights=False)
+    .sel(index=0, drop=True)
+    .assign_coords(gas="CO2", sector="OAE_CDR")
+    * ind_co2_dimensions
 )
 
-# %% [markdown]
-# **TODO** We might want to try to give the OAE CDR negative emissions some seasonality that correlates with industry emissions. Unfortunately, the industry co2 seasonality is different between regions (compare `ind_co2.sel(lon=slice(0, 20), lat=slice(40, 20)).mean(["year", "gas", "lat", "lon"]).plot()` (Europe) to `ind_co2.sel(lon=slice(0, 20), lat=slice(-10, 30)).mean(["year", "gas", "lat", "lon"]).plot()` (Africa))
-#
+# %%
+plot_map(oae_cdr.sel(year=2050, month=1).assign_attrs(long_name="OAE CDR emissions"))
 
 # %% [markdown]
-# # DACCS and Industrial CDR
+# ## DACCS and Industrial CDR
 #
 # Combine renewable potential from GaSP, Global Wind and Solar Atlas with CO2 storage potential
 #
@@ -219,11 +231,6 @@ def read_co2_storage_potential(smooth=True):
     title="DACCS suitability",
 )
 
-# %%
-ind_co2_dimensions = xr.ones_like(ind_co2.drop_vars("sector"))
-ind_co2_seasonality = ind_co2.sum(["gas", "year"])
-ind_co2_seasonality /= ind_co2_seasonality.sum(["lat", "lon"]).mean("month")
-
 # %%
 # industry CDR is composition of daccs potential and availability of industrial co2 emissions
 ind_cdr = (daccs_potential * ind_co2).assign_coords(sector="IND_CDR")
@@ -255,7 +262,7 @@ def read_co2_storage_potential(smooth=True):
     xr.concat(
         [
             ind_cdr,
-            # oae_cdr,
+            oae_cdr,
             # oae_co2, # Part of other emissions
             dac_cdr,
         ],