Merge pull request #663 from PyPSA/post-merge-param

Post-merge `snakemake.params`

config/test/config.myopic.yaml

@@ -31,6 +31,14 @@ snapshots:
   end: "2013-03-08"
 
 electricity:
+  co2limit: 100.e+6
+
+  extendable_carriers:
+    Generator: [OCGT]
+    StorageUnit: [battery]
+    Store: [H2]
+    Link: [H2 pipeline]
+
   renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
 
 atlite:

config/test/config.overnight.yaml

@@ -28,6 +28,14 @@ snapshots:
   end: "2013-03-08"
 
 electricity:
+  co2limit: 100.e+6
+
+  extendable_carriers:
+    Generator: [OCGT]
+    StorageUnit: [battery]
+    Store: [H2]
+    Link: [H2 pipeline]
+
   renewable_carriers: [solar, onwind, offwind-ac, offwind-dc]
 
 atlite:

doc/release_notes.rst

@@ -10,6 +10,7 @@ Release Notes
 Upcoming Release
 ================
 
+* ``param:`` section in rule definition are added to track changed settings in ``config.yaml``. The goal is to automatically re-execute rules whose parameters have changed. See `Non-file parameters for rules <https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#non-file-parameters-for-rules>`_ in the snakemake documentation.
 
 * **Important:** The configuration files are now located in the ``config`` directory. This counts for ``config.default.yaml``, ``config.yaml`` as well as the test configuration files which are now located in ``config/test``. Config files that are still in the root directory will be ignored.
 

rules/build_electricity.smk

@@ -19,6 +19,10 @@ if config["enable"].get("prepare_links_p_nom", False):
 
 
 rule build_electricity_demand:
+    params:
+        snapshots=config["snapshots"],
+        countries=config["countries"],
+        load=config["load"],
     input:
         ancient("data/load_raw.csv"),
     output:
@@ -34,6 +38,10 @@ rule build_electricity_demand:
 
 
 rule build_powerplants:
+    params:
+        powerplants_filter=config["electricity"]["powerplants_filter"],
+        custom_powerplants=config["electricity"]["custom_powerplants"],
+        countries=config["countries"],
     input:
         base_network=RESOURCES + "networks/base.nc",
         custom_powerplants="data/custom_powerplants.csv",
@@ -79,6 +87,8 @@ rule base_network:
 
 
 rule build_shapes:
+    params:
+        countries=config["countries"],
     input:
         naturalearth=ancient("data/bundle/naturalearth/ne_10m_admin_0_countries.shp"),
         eez=ancient("data/bundle/eez/World_EEZ_v8_2014.shp"),
@@ -104,6 +114,8 @@ rule build_shapes:
 
 
 rule build_bus_regions:
+    params:
+        countries=config["countries"],
     input:
         country_shapes=RESOURCES + "country_shapes.geojson",
         offshore_shapes=RESOURCES + "offshore_shapes.geojson",
@@ -125,6 +137,9 @@ rule build_bus_regions:
 if config["enable"].get("build_cutout", False):
 
     rule build_cutout:
+        params:
+            snapshots=config["snapshots"],
+            cutouts=config["atlite"]["cutouts"],
         input:
             regions_onshore=RESOURCES + "regions_onshore.geojson",
             regions_offshore=RESOURCES + "regions_offshore.geojson",
@@ -186,6 +201,8 @@ rule build_ship_raster:
 
 
 rule build_renewable_profiles:
+    params:
+        renewable=config["renewable"],
     input:
         base_network=RESOURCES + "networks/base.nc",
         corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"),
@@ -235,6 +252,9 @@ rule build_renewable_profiles:
 
 
 rule build_hydro_profile:
+    params:
+        hydro=config["renewable"]["hydro"],
+        countries=config["countries"],
     input:
         country_shapes=RESOURCES + "country_shapes.geojson",
         eia_hydro_generation="data/eia_hydro_annual_generation.csv",
@@ -252,6 +272,14 @@ rule build_hydro_profile:
 
 
 rule add_electricity:
+    params:
+        length_factor=config["lines"]["length_factor"],
+        scaling_factor=config["load"]["scaling_factor"],
+        countries=config["countries"],
+        renewable=config["renewable"],
+        electricity=config["electricity"],
+        conventional=config.get("conventional", {}),
+        costs=config["costs"],
     input:
         **{
             f"profile_{tech}": RESOURCES + f"profile_{tech}.nc"
@@ -287,6 +315,15 @@ rule add_electricity:
 
 
 rule simplify_network:
+    params:
+        simplify_network=config["clustering"]["simplify_network"],
+        aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
+        focus_weights=config.get("focus_weights", None),
+        renewable_carriers=config["electricity"]["renewable_carriers"],
+        max_hours=config["electricity"]["max_hours"],
+        length_factor=config["lines"]["length_factor"],
+        p_max_pu=config["links"].get("p_max_pu", 1.0),
+        costs=config["costs"],
     input:
         network=RESOURCES + "networks/elec.nc",
         tech_costs=COSTS,
@@ -312,6 +349,16 @@ rule simplify_network:
 
 
 rule cluster_network:
+    params:
+        cluster_network=config["clustering"]["cluster_network"],
+        aggregation_strategies=config["clustering"].get("aggregation_strategies", {}),
+        custom_busmap=config["enable"].get("custom_busmap", False),
+        focus_weights=config.get("focus_weights", None),
+        renewable_carriers=config["electricity"]["renewable_carriers"],
+        conventional_carriers=config["electricity"].get("conventional_carriers", []),
+        max_hours=config["electricity"]["max_hours"],
+        length_factor=config["lines"]["length_factor"],
+        costs=config["costs"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson",
@@ -343,6 +390,10 @@ rule cluster_network:
 
 
 rule add_extra_components:
+    params:
+        extendable_carriers=config["electricity"]["extendable_carriers"],
+        max_hours=config["electricity"]["max_hours"],
+        costs=config["costs"],
     input:
         network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc",
         tech_costs=COSTS,
@@ -362,6 +413,14 @@ rule add_extra_components:
 
 
 rule prepare_network:
+    params:
+        links=config["links"],
+        lines=config["lines"],
+        co2base=config["electricity"]["co2base"],
+        co2limit=config["electricity"]["co2limit"],
+        gaslimit=config["electricity"].get("gaslimit"),
+        max_hours=config["electricity"]["max_hours"],
+        costs=config["costs"],
     input:
         RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc",
         tech_costs=COSTS,

rules/build_sector.smk

@@ -140,6 +140,8 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]):
 
 
 rule build_heat_demands:
+    params:
+        snapshots=config["snapshots"],
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -160,6 +162,8 @@ rule build_heat_demands:
 
 
 rule build_temperature_profiles:
+    params:
+        snapshots=config["snapshots"],
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -181,6 +185,8 @@ rule build_temperature_profiles:
 
 
 rule build_cop_profiles:
+    params:
+        heat_pump_sink_T=config["sector"]["heat_pump_sink_T"],
     input:
         temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc",
         temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc",
@@ -208,6 +214,9 @@ rule build_cop_profiles:
 
 
 rule build_solar_thermal_profiles:
+    params:
+        snapshots=config["snapshots"],
+        solar_thermal=config["solar_thermal"],
     input:
         pop_layout=RESOURCES + "pop_layout_{scope}.nc",
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
@@ -228,6 +237,9 @@ rule build_solar_thermal_profiles:
 
 
 rule build_energy_totals:
+    params:
+        countries=config["countries"],
+        energy=config["energy"],
     input:
         nuts3_shapes=RESOURCES + "nuts3_shapes.geojson",
         co2="data/eea/UNFCCC_v23.csv",
@@ -253,6 +265,8 @@ rule build_energy_totals:
 
 
 rule build_biomass_potentials:
+    params:
+        biomass=config["biomass"],
     input:
         enspreso_biomass=HTTP.remote(
             "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx",
@@ -315,6 +329,10 @@ if not config["sector"]["biomass_transport"]:
 if config["sector"]["regional_co2_sequestration_potential"]["enable"]:
 
     rule build_sequestration_potentials:
+        params:
+            sequestration_potential=config["sector"][
+                "regional_co2_sequestration_potential"
+            ],
         input:
             sequestration_potential=HTTP.remote(
                 "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson",
@@ -368,6 +386,8 @@ rule build_salt_cavern_potentials:
 
 
 rule build_ammonia_production:
+    params:
+        countries=config["countries"],
     input:
         usgs="data/myb1-2017-nitro.xls",
     output:
@@ -386,6 +406,9 @@ rule build_ammonia_production:
 
 
 rule build_industry_sector_ratios:
+    params:
+        industry=config["industry"],
+        ammonia=config["sector"].get("ammonia", False),
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
         idees="data/jrc-idees-2015",
@@ -405,6 +428,9 @@ rule build_industry_sector_ratios:
 
 
 rule build_industrial_production_per_country:
+    params:
+        industry=config["industry"],
+        countries=config["countries"],
     input:
         ammonia_production=RESOURCES + "ammonia_production.csv",
         jrc="data/jrc-idees-2015",
@@ -426,6 +452,8 @@ rule build_industrial_production_per_country:
 
 
 rule build_industrial_production_per_country_tomorrow:
+    params:
+        industry=config["industry"],
     input:
         industrial_production_per_country=RESOURCES
         + "industrial_production_per_country.csv",
@@ -450,6 +478,9 @@ rule build_industrial_production_per_country_tomorrow:
 
 
 rule build_industrial_distribution_key:
+    params:
+        hotmaps_locate_missing=config["industry"].get("hotmaps_locate_missing", False),
+        countries=config["countries"],
     input:
         regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson",
         clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
@@ -524,6 +555,9 @@ rule build_industrial_energy_demand_per_node:
 
 
 rule build_industrial_energy_demand_per_country_today:
+    params:
+        countries=config["countries"],
+        industry=config["industry"],
     input:
         jrc="data/jrc-idees-2015",
         ammonia_production=RESOURCES + "ammonia_production.csv",
@@ -570,6 +604,9 @@ rule build_industrial_energy_demand_per_node_today:
 if config["sector"]["retrofitting"]["retro_endogen"]:
 
     rule build_retro_cost:
+        params:
+            retrofitting=config["sector"]["retrofitting"],
+            countries=config["countries"],
         input:
             building_stock="data/retro/data_building_stock.csv",
             data_tabula="data/retro/tabula-calculator-calcsetbuilding.csv",
@@ -640,6 +677,9 @@ rule build_shipping_demand:
 
 
 rule build_transport_demand:
+    params:
+        snapshots=config["snapshots"],
+        sector=config["sector"],
     input:
         clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv",
         pop_weighted_energy_totals=RESOURCES
@@ -666,6 +706,18 @@ rule build_transport_demand:
 
 rule prepare_sector_network:
     params:
+        co2_budget=config["co2_budget"],
+        conventional_carriers=config["existing_capacities"]["conventional_carriers"],
+        foresight=config["foresight"],
+        costs=config["costs"],
+        sector=config["sector"],
+        industry=config["industry"],
+        pypsa_eur=config["pypsa_eur"],
+        length_factor=config["lines"]["length_factor"],
+        planning_horizons=config["scenario"]["planning_horizons"],
+        countries=config["countries"],
+        emissions_scope=config["energy"]["emissions"],
+        eurostat_report_year=config["energy"]["eurostat_report_year"],
         RDIR=RDIR,
     input:
         **build_retro_cost_output,

rules/postprocess.smk

@@ -9,6 +9,9 @@ localrules:
 
 
 rule plot_network:
+    params:
+        foresight=config["foresight"],
+        plotting=config["plotting"],
     input:
         overrides="data/override_component_attrs",
         network=RESULTS
@@ -67,6 +70,10 @@ rule copy_conda_env:
 
 rule make_summary:
     params:
+        foresight=config["foresight"],
+        costs=config["costs"],
+        snapshots=config["snapshots"],
+        scenario=config["scenario"],
         RDIR=RDIR,
     input:
         overrides="data/override_component_attrs",
@@ -114,6 +121,10 @@ rule make_summary:
 
 rule plot_summary:
     params:
+        countries=config["countries"],
+        planning_horizons=config["scenario"]["planning_horizons"],
+        sector_opts=config["scenario"]["sector_opts"],
+        plotting=config["plotting"],
         RDIR=RDIR,
     input:
         costs=RESULTS + "csvs/costs.csv",

rules/retrieve.smk

@@ -19,6 +19,8 @@ if config["enable"].get("retrieve_databundle", True):
         datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"])
 
     rule retrieve_databundle:
+        params:
+            tutorial=config["tutorial"],
         output:
             expand("data/bundle/{file}", file=datafiles),
         log: