Merge pull request #823 from virio-andreyana/add-param

Add `snakemake.params` to track config updates

Snakefile

@@ -145,6 +145,9 @@ rule plot_all_summaries:
 if config["enable"].get("retrieve_databundle", True):
 
     rule retrieve_databundle_light:
+        params:
+            countries=config["countries"],
+            tutorial=config["tutorial"],
         output:  #expand(directory('{file}') if isdir('{file}') else '{file}', file=datafiles)
             expand("{file}", file=datafiles_retrivedatabundle(config)),
             directory("data/landcover"),
@@ -159,6 +162,8 @@ if config["enable"].get("retrieve_databundle", True):
 if config["enable"].get("download_osm_data", True):
 
     rule download_osm_data:
+        params:
+            countries=config["countries"],
         output:
             cables="resources/" + RDIR + "osm/raw/all_raw_cables.geojson",
             generators="resources/" + RDIR + "osm/raw/all_raw_generators.geojson",
@@ -174,6 +179,9 @@ if config["enable"].get("download_osm_data", True):
 
 
 rule clean_osm_data:
+    params:
+        crs=config["crs"],
+        clean_osm_data_options=config["clean_osm_data_options"],
     input:
         cables="resources/" + RDIR + "osm/raw/all_raw_cables.geojson",
         generators="resources/" + RDIR + "osm/raw/all_raw_generators.geojson",
@@ -196,6 +204,10 @@ rule clean_osm_data:
 
 
 rule build_osm_network:
+    params:
+        build_osm_network=config.get("build_osm_network", {}),
+        countries=config["countries"],
+        crs=config["crs"],
     input:
         generators="resources/" + RDIR + "osm/clean/all_clean_generators.geojson",
         lines="resources/" + RDIR + "osm/clean/all_clean_lines.geojson",
@@ -217,6 +229,10 @@ rule build_osm_network:
 
 
 rule build_shapes:
+    params:
+        build_shape_options=config["build_shape_options"],
+        crs=config["crs"],
+        countries=config["countries"],
     input:
         # naturalearth='data/bundle/naturalearth/ne_10m_admin_0_countries.shp',
         # eez='data/bundle/eez/World_EEZ_v8_2014.shp',
@@ -241,6 +257,15 @@ rule build_shapes:
 
 
 rule base_network:
+    params:
+        voltages=config["electricity"]["voltages"],
+        transformers=config["transformers"],
+        snapshots=config["snapshots"],
+        links=config["links"],
+        lines=config["lines"],
+        hvdc_as_lines=config["electricity"]["hvdc_as_lines"],
+        countries=config["countries"],
+        base_network=config["base_network"],
     input:
         osm_buses="resources/" + RDIR + "base_network/all_buses_build_network.csv",
         osm_lines="resources/" + RDIR + "base_network/all_lines_build_network.csv",
@@ -266,6 +291,10 @@ rule base_network:
 
 
 rule build_bus_regions:
+    params:
+        alternative_clustering=config["cluster_options"]["alternative_clustering"],
+        area_crs=config["crs"]["area_crs"],
+        countries=config["countries"],
     input:
         country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson",
         offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson",
@@ -312,6 +341,9 @@ if config["enable"].get("build_cutout", False):
     terminate_if_cutout_exists(config)
 
     rule build_cutout:
+        params:
+            snapshots=config["snapshots"],
+            cutouts=config["atlite"]["cutouts"],
         input:
             onshore_shapes="resources/" + RDIR + "shapes/country_shapes.geojson",
             offshore_shapes="resources/" + RDIR + "shapes/offshore_shapes.geojson",
@@ -331,6 +363,8 @@ if config["enable"].get("build_cutout", False):
 if config["enable"].get("build_natura_raster", False):
 
     rule build_natura_raster:
+        params:
+            area_crs=config["crs"]["area_crs"],
         input:
             shapefiles_land="data/landcover",
             cutouts=expand("cutouts/" + CDIR + "{cutouts}.nc", **config["atlite"]),
@@ -376,6 +410,10 @@ if config["enable"].get("retrieve_cost_data", True):
 
 
 rule build_demand_profiles:
+    params:
+        snapshots=config["snapshots"],
+        load_options=config["load_options"],
+        countries=config["countries"],
     input:
         base_network="networks/" + RDIR + "base.nc",
         regions="resources/" + RDIR + "bus_regions/regions_onshore.geojson",
@@ -399,6 +437,11 @@ rule build_demand_profiles:
 
 
 rule build_renewable_profiles:
+    params:
+        crs=config["crs"],
+        renewable=config["renewable"],
+        countries=config["countries"],
+        alternative_clustering=config["cluster_options"]["alternative_clustering"],
     input:
         natura="resources/" + RDIR + "natura.tiff",
         copernicus="data/copernicus/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif",
@@ -431,6 +474,12 @@ rule build_renewable_profiles:
 
 
 rule build_powerplants:
+    params:
+        geo_crs=config["crs"]["geo_crs"],
+        countries=config["countries"],
+        gadm_layer_id=config["build_shape_options"]["gadm_layer_id"],
+        alternative_clustering=config["cluster_options"]["alternative_clustering"],
+        powerplants_filter=config["electricity"]["powerplants_filter"],
     input:
         base_network="networks/" + RDIR + "base.nc",
         pm_config="configs/powerplantmatching_config.yaml",
@@ -456,6 +505,14 @@ rule build_powerplants:
 
 
 rule add_electricity:
+    params:
+        countries=config["countries"],
+        costs=config["costs"],
+        conventional=config.get("conventional", {}),
+        electricity=config["electricity"],
+        alternative_clustering=config["cluster_options"]["alternative_clustering"],
+        renewable=config["renewable"],
+        length_factor=config["lines"]["length_factor"],
     input:
         **{
             f"profile_{tech}": "resources/"
@@ -494,6 +551,17 @@ rule add_electricity:
 
 
 rule simplify_network:
+    params:
+        renewable=config["renewable"],
+        geo_crs=config["crs"]["geo_crs"],
+        cluster_options=config["cluster_options"],
+        countries=config["countries"],
+        build_shape_options=config["build_shape_options"],
+        electricity=config["electricity"],
+        costs=config["costs"],
+        lines_types=config["lines"]["types"],
+        lines_length_factor=config["lines"]["length_factor"],
+        focus_weights=config.get("focus_weights", None),
     input:
         network="networks/" + RDIR + "elec.nc",
         tech_costs=COSTS,
@@ -525,6 +593,17 @@ rule simplify_network:
 if config["augmented_line_connection"].get("add_to_snakefile", False) == True:
 
     rule cluster_network:
+        params:
+            build_shape_options=config["build_shape_options"],
+            electricity=config["electricity"],
+            costs=config["costs"],
+            length_factor=config["lines"]["length_factor"],
+            renewable=config["renewable"],
+            geo_crs=config["crs"]["geo_crs"],
+            countries=config["countries"],
+            cluster_options=config["cluster_options"],
+            focus_weights=config.get("focus_weights", None),
+            #custom_busmap=config["enable"].get("custom_busmap", False)
         input:
             network="networks/" + RDIR + "elec_s{simpl}.nc",
             country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson",
@@ -568,6 +647,12 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == True:
             "scripts/cluster_network.py"
 
     rule augmented_line_connections:
+        params:
+            lines=config["lines"],
+            augmented_line_connection=config["augmented_line_connection"],
+            hvdc_as_lines=config["electricity"]["hvdc_as_lines"],
+            electricity=config["electricity"],
+            costs=config["costs"],
         input:
             tech_costs=COSTS,
             network="networks/" + RDIR + "elec_s{simpl}_{clusters}_pre_augmentation.nc",
@@ -593,6 +678,16 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == True:
 if config["augmented_line_connection"].get("add_to_snakefile", False) == False:
 
     rule cluster_network:
+        params:
+            build_shape_options=config["build_shape_options"],
+            electricity=config["electricity"],
+            costs=config["costs"],
+            length_factor=config["lines"]["length_factor"],
+            renewable=config["renewable"],
+            geo_crs=config["crs"]["geo_crs"],
+            countries=config["countries"],
+            gadm_layer_id=config["build_shape_options"]["gadm_layer_id"],
+            cluster_options=config["cluster_options"],
         input:
             network="networks/" + RDIR + "elec_s{simpl}.nc",
             country_shapes="resources/" + RDIR + "shapes/country_shapes.geojson",
@@ -654,6 +749,12 @@ rule add_extra_components:
 
 
 rule prepare_network:
+    params:
+        links=config["links"],
+        lines=config["lines"],
+        s_max_pu=config["lines"]["s_max_pu"],
+        electricity=config["electricity"],
+        costs=config["costs"],
     input:
         "networks/" + RDIR + "elec_s{simpl}_{clusters}_ec.nc",
         tech_costs=COSTS,
@@ -695,6 +796,9 @@ def memory(w):
 if config["monte_carlo"]["options"].get("add_to_snakefile", False) == False:
 
     rule solve_network:
+        params:
+            solving=config["solving"],
+            augmented_line_connection=config["augmented_line_connection"],
         input:
             "networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
         output:
@@ -726,6 +830,8 @@ if config["monte_carlo"]["options"].get("add_to_snakefile", False) == False:
 if config["monte_carlo"]["options"].get("add_to_snakefile", False) == True:
 
     rule monte_carlo:
+        params:
+            monte_carlo=config["monte_carlo"],
         input:
             "networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc",
         output:
@@ -756,6 +862,9 @@ if config["monte_carlo"]["options"].get("add_to_snakefile", False) == True:
             ),
 
     rule solve_network:
+        params:
+            solving=config["solving"],
+            augmented_line_connection=config["augmented_line_connection"],
         input:
             "networks/" + RDIR + "elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{unc}.nc",
         output:
@@ -817,6 +926,11 @@ def input_make_summary(w):
 
 
 rule make_summary:
+    params:
+        electricity=config["electricity"],
+        costs=config["costs"],
+        ll=config["scenario"]["ll"],
+        scenario=config["scenario"],
     input:
         input_make_summary,
         tech_costs=COSTS,
@@ -852,6 +966,10 @@ rule plot_summary:
 
 
 rule plot_network:
+    params:
+        electricity=config["electricity"],
+        costs=config["costs"],
+        plotting=config["plotting"],
     input:
         network="results/"
         + RDIR
@@ -894,6 +1012,12 @@ rule build_test_configs:
 
 
 rule make_statistics:
+    params:
+        countries=config["countries"],
+        renewable_carriers=config["electricity"]["renewable_carriers"],
+        renewable=config["renewable"],
+        crs=config["crs"],
+        scenario=config["scenario"],
     output:
         stats="results/" + RDIR + "stats.csv",
     threads: 1

doc/release_notes.rst

@@ -14,6 +14,8 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t
 
 **New Features and major Changes**
 
+* Add params: section in rule definition to keep track of changed settings in config.yaml. `PR #823 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/823>`__
+
 * Fix Natural Gas implementation in "add_electricity" to avoid "Natural Gas" to be filtered out `PR #797 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/797>`__
 
 * Improve network simplification routine to account for representation HVDC as Line component `PR #743 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/743>`__

scripts/_helpers.py

@@ -152,7 +152,7 @@ def load_network(import_name=None, custom_components=None):
         As in pypsa.Network(import_name)
     custom_components : dict
         Dictionary listing custom components.
-        For using ``snakemake.config["override_components"]``
+        For using ``snakemake.params.override_components"]``
         in ``config.yaml`` define:
 
         .. code:: yaml
@@ -200,7 +200,9 @@ def pdbcast(v, h):
     )
 
 
-def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True):
+def load_network_for_plots(
+    fn, tech_costs, cost_config, elec_config, combine_hydro_ps=True
+):
     import pypsa
     from add_electricity import load_costs, update_transmission_costs
 
@@ -228,7 +230,7 @@ def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True):
     # n.storage_units.loc[bus_carrier == "heat","carrier"] = "water tanks"
 
     Nyears = n.snapshot_weightings.objective.sum() / 8760.0
-    costs = load_costs(Nyears, tech_costs, config["costs"], config["electricity"])
+    costs = load_costs(tech_costs, cost_config, elec_config, Nyears)
     update_transmission_costs(n, costs)
 
     return n