use snakemake.params to auto re-run due to config changes

Snakefile

@@ -63,6 +63,7 @@ if not config.get('tutorial', False):
 if config['enable'].get('retrieve_databundle', True):
     rule retrieve_databundle:
         output: expand('data/bundle/{file}', file=datafiles)
+        params: tutorial = config["tutorial"]
         log: "logs/retrieve_databundle.log"
         script: 'scripts/retrieve_databundle.py'
 
@@ -76,6 +77,10 @@ rule retrieve_load_data:
 rule build_load_data:
     input: "data/load_raw.csv"
     output: "resources/load.csv"
+    params:
+        load=config["load"],
+        countries=config["countries"],
+        snapshots=config["snapshots"]
     log: "logs/build_load_data.log"
     script: 'scripts/build_load_data.py'
 
@@ -85,6 +90,9 @@ rule build_powerplants:
         base_network="networks/base.nc",
         custom_powerplants="data/custom_powerplants.csv"
     output: "resources/powerplants.csv"
+    params: 
+        powerplants_filter = config["electricity"]["powerplants_filter"],
+        custom_powerplants = config["electricity"]["custom_powerplants"]
     log: "logs/build_powerplants.log"
     threads: 1
     resources: mem_mb=500
@@ -105,6 +113,13 @@ rule base_network:
         offshore_shapes='resources/offshore_shapes.geojson',
         europe_shape='resources/europe_shape.geojson'
     output: "networks/base.nc"
+    params:
+        voltages=config["electricity"]["voltages"],
+        lines=config["lines"],
+        links=config["links"],
+        transformers=config["transformers"],
+        countries=config["countries"],
+        snapshots=config["snapshots"]
     log: "logs/base_network.log"
     benchmark: "benchmarks/base_network"
     threads: 1
@@ -126,6 +141,8 @@ rule build_shapes:
         offshore_shapes='resources/offshore_shapes.geojson',
         europe_shape='resources/europe_shape.geojson',
         nuts3_shapes='resources/nuts3_shapes.geojson'
+    params:
+        countries=config["countries"]
     log: "logs/build_shapes.log"
     threads: 1
     resources: mem_mb=500
@@ -140,6 +157,8 @@ rule build_bus_regions:
     output:
         regions_onshore="resources/regions_onshore.geojson",
         regions_offshore="resources/regions_offshore.geojson"
+    params:
+        countries=config["countries"]
     log: "logs/build_bus_regions.log"
     threads: 1
     resources: mem_mb=1000
@@ -151,6 +170,9 @@ if config['enable'].get('build_cutout', False):
             regions_onshore="resources/regions_onshore.geojson",
             regions_offshore="resources/regions_offshore.geojson"
         output: "cutouts/{cutout}.nc"
+        params:
+            cutout_params=lambda w: config['atlite']['cutouts'][w.cutout],
+            snapshots=config["snapshots"]
         log: "logs/build_cutout/{cutout}.log"
         benchmark: "benchmarks/build_cutout_{cutout}"
         threads: ATLITE_NPROCESSES
@@ -196,7 +218,11 @@ rule build_renewable_profiles:
                                    if w.technology in ('onwind', 'solar')
                                    else "resources/regions_offshore.geojson"),
         cutout=lambda w: "cutouts/" + config["renewable"][w.technology]['cutout'] + ".nc"
-    output: profile="resources/profile_{technology}.nc",
+    output:
+        profile="resources/profile_{technology}.nc",
+    params:
+        atlite=config["atlite"],
+        tech_params=lambda w: config["renewable"][w.technology]
     log: "logs/build_renewable_profile_{technology}.log"
     benchmark: "benchmarks/build_renewable_profiles_{technology}"
     threads: ATLITE_NPROCESSES
@@ -211,6 +237,9 @@ if 'hydro' in config['renewable'].keys():
             eia_hydro_generation='data/bundle/EIA_hydro_generation_2000_2014.csv',
             cutout="cutouts/" + config["renewable"]['hydro']['cutout'] + ".nc"
         output: 'resources/profile_hydro.nc'
+        params:
+            countries=config["countries"],
+            hydro=config["renewable"]["hydro"]
         log: "logs/build_hydro_profile.log"
         resources: mem_mb=5000
         script: 'scripts/build_hydro_profile.py'
@@ -229,6 +258,14 @@ rule add_electricity:
         **{f"profile_{tech}": f"resources/profile_{tech}.nc"
            for tech in config['renewable']}
     output: "networks/elec.nc"
+    params:
+        costs=config["costs"],
+        electricity=config["electricity"],
+        countries=config["countries"],
+        scaling_factor=config["load"]["scaling_factor"],
+        length_factor=config["lines"]["length_factor"],
+        renewable=config["renewable"],
+        plotting=config["plotting"]
     log: "logs/add_electricity.log"
     benchmark: "benchmarks/add_electricity"
     threads: 1
@@ -248,6 +285,15 @@ rule simplify_network:
         regions_offshore="resources/regions_offshore_elec_s{simpl}.geojson",
         busmap='resources/busmap_elec_s{simpl}.csv',
         connection_costs='resources/connection_costs_s{simpl}.csv'
+    params:
+        electricity=config["electricity"],
+        costs=config["costs"],
+        renewable=config["renewable"],
+        length_factor=config['lines']['length_factor'],
+        p_max_pu=config["links"]['p_max_pu'],
+        solver_name=config["solving"]["solver"]["name"],
+        to_substations=config["clustering"]["simplify"]["to_substations"],
+        focus_weights=config["clustering"]["focus_weights"]
     log: "logs/simplify_network/elec_s{simpl}.log"
     benchmark: "benchmarks/simplify_network/elec_s{simpl}"
     threads: 1
@@ -270,6 +316,14 @@ rule cluster_network:
         regions_offshore="resources/regions_offshore_elec_s{simpl}_{clusters}.geojson",
         busmap="resources/busmap_elec_s{simpl}_{clusters}.csv",
         linemap="resources/linemap_elec_s{simpl}_{clusters}.csv"
+    params:
+        solver_name=config["solving"]["solver"]["name"],
+        renewable=config["renewable"],
+        length_factor=config["lines"]["length_factor"],
+        custom_busmap=config["enable"]["custom_busmap"],
+        costs=config["costs"],
+        electricity=config["electricity"],
+        focus_weights=config["clustering"]["focus_weights"]
     log: "logs/cluster_network/elec_s{simpl}_{clusters}.log"
     benchmark: "benchmarks/cluster_network/elec_s{simpl}_{clusters}"
     threads: 1
@@ -282,6 +336,10 @@ rule add_extra_components:
         network='networks/elec_s{simpl}_{clusters}.nc',
         tech_costs=COSTS,
     output: 'networks/elec_s{simpl}_{clusters}_ec.nc'
+    params:
+        costs=config["costs"],
+        electricity=config["electricity"],
+        plotting=config["plotting"]
     log: "logs/add_extra_components/elec_s{simpl}_{clusters}.log"
     benchmark: "benchmarks/add_extra_components/elec_s{simpl}_{clusters}_ec"
     threads: 1
@@ -292,6 +350,13 @@ rule add_extra_components:
 rule prepare_network:
     input: 'networks/elec_s{simpl}_{clusters}_ec.nc', tech_costs=COSTS
     output: 'networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc'
+    params:
+        electricity=config["electricity"],
+        s_max_pu=config["lines"]["s_max_pu"],
+        s_nom_max=config["lines"]["s_nom_max"],
+        p_nom_max=config["links"]["p_nom_max"],
+        solver_name=config["solving"]["solver"]["name"],
+        costs=config["costs"]
     log: "logs/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.log"
     benchmark: "benchmarks/prepare_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}"
     threads: 1
@@ -320,6 +385,17 @@ def memory(w):
 rule solve_network:
     input: "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"
     output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"
+    params:
+        agg_p_nom_limits=config["electricity"]["agg_p_nom_limits"],
+        BAU_mincapacities=config['electricity']['BAU_mincapacities'],
+        SAFE_reservemargin=config['electricity']['SAFE_reservemargin'],
+        conv_techs=config['electricity']['conventional_carriers'],        
+        solver_options=config["solving"]["solver"],
+        track_iterations=config["solving"]["options"]["track_iterations"],
+        min_iterations=config["solving"]["options"]["min_iterations"],
+        max_iterations=config["solving"]["options"]["max_iterations"],
+        skip_iterations=config["solving"]["options"]["skip_iterations"],
+        solving=config["solving"]
     log:
         solver=normpath("logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_solver.log"),
         python="logs/solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log",
@@ -336,6 +412,17 @@ rule solve_operations_network:
         unprepared="networks/elec_s{simpl}_{clusters}_ec.nc",
         optimized="results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"
     output: "results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op.nc"
+    params: 
+        agg_p_nom_limits=config["electricity"]["agg_p_nom_limits"],
+        BAU_mincapacities=config['electricity']['BAU_mincapacities'],
+        SAFE_reservemargin=config['electricity']['SAFE_reservemargin'],
+        conv_techs=config['electricity']['conventional_carriers'],        
+        solver_options=config["solving"]["solver"],
+        track_iterations=config["solving"]["options"]["track_iterations"],
+        min_iterations=config["solving"]["options"]["min_iterations"],
+        max_iterations=config["solving"]["options"]["max_iterations"],
+        skip_iterations=config["solving"]["options"]["skip_iterations"],
+        solving=config["solving"]
     log:
         solver=normpath("logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_solver.log"),
         python="logs/solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log",
@@ -354,6 +441,10 @@ rule plot_network:
     output:
         only_map="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}.{ext}",
         ext="results/plots/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_ext.{ext}"
+    params:
+        plotting=config["plotting"],
+        electricity=config["electricity"],
+        costs=config["costs"]
     log: "logs/plot_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{attr}_{ext}.log"
     script: "scripts/plot_network.py"
 
@@ -376,6 +467,9 @@ def input_make_summary(w):
 rule make_summary:
     input: input_make_summary
     output: directory("results/summaries/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}")
+    params: 
+        electricity=config["electricity"],
+        costs=config["costs"]
     log: "logs/make_summary/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_{country}.log",
     script: "scripts/make_summary.py"
 

config.default.yaml

@@ -20,6 +20,7 @@ scenario:
 countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK']
 
 clustering:
+  focus_weights: false
   simplify:
     to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
 
@@ -41,7 +42,9 @@ electricity:
   voltages: [220., 300., 380.]
   co2limit: 7.75e+7 # 0.05 * 3.1e9*0.5
   co2base: 1.487e+9
-  agg_p_nom_limits: data/agg_p_nom_minmax.csv
+  agg_p_nom_limits: false
+  BAU_mincapacities: false
+  SAFE_reservemargin: false
 
   extendable_carriers:
     Generator: []

config.tutorial.yaml

@@ -20,6 +20,7 @@ scenario:
 countries: ['DE']
 
 clustering:
+  focus_weights: false
   simplify:
     to_substations: false # network is simplified to nodes with positive or negative power injection (i.e. substations or offwind connections)
 
@@ -40,6 +41,9 @@ enable:
 electricity:
   voltages: [220., 300., 380.]
   co2limit: 100.e+6
+  agg_p_nom_limits: false
+  BAU_mincapacities: false
+  SAFE_reservemargin: false
 
   extendable_carriers:
     Generator: [OCGT]

scripts/_helpers.py

@@ -96,7 +96,7 @@ def pdbcast(v, h):
                         index=v.index, columns=h.index)
 
 
-def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True):
+def load_network_for_plots(fn, tech_costs, params, combine_hydro_ps=True):
     import pypsa
     from add_electricity import update_transmission_costs, load_costs
 
@@ -120,7 +120,7 @@ def load_network_for_plots(fn, tech_costs, config, combine_hydro_ps=True):
     # n.storage_units.loc[bus_carrier == "heat","carrier"] = "water tanks"
 
     Nyears = n.snapshot_weightings.objective.sum() / 8760.
-    costs = load_costs(Nyears, tech_costs, config['costs'], config['electricity'])
+    costs = load_costs(tech_costs, params['costs'], params['electricity'], Nyears)
     update_transmission_costs(n, costs)
 
     return n