From 1c023053dd78934d17703e03299fd506772f114e Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 8 Jan 2024 14:59:19 -0800 Subject: [PATCH 01/42] Fix import of pandas testing --- tests/test_density.py | 2 +- tests/test_modelchain.py | 2 +- tests/test_power_curves.py | 2 +- tests/test_power_output.py | 2 +- tests/test_temperature.py | 2 +- tests/test_tools.py | 2 +- tests/test_turbine_cluster_modelchain.py | 2 +- tests/test_wake_losses.py | 2 +- tests/test_wind_speed.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_density.py b/tests/test_density.py index 68556f4d..1f6964d4 100644 --- a/tests/test_density.py +++ b/tests/test_density.py @@ -5,7 +5,7 @@ import pandas as pd import numpy as np -from pandas.util.testing import assert_series_equal +from pandas.testing import assert_series_equal from numpy.testing import assert_allclose from windpowerlib.density import barometric, ideal_gas diff --git a/tests/test_modelchain.py b/tests/test_modelchain.py index 5b2c8b2f..a9b8f657 100644 --- a/tests/test_modelchain.py +++ b/tests/test_modelchain.py @@ -7,7 +7,7 @@ import pandas as pd import numpy as np import pytest -from pandas.util.testing import assert_series_equal +from pandas.testing import assert_series_equal import windpowerlib.wind_turbine as wt import windpowerlib.modelchain as mc diff --git a/tests/test_power_curves.py b/tests/test_power_curves.py index ccf296ab..0eed069d 100644 --- a/tests/test_power_curves.py +++ b/tests/test_power_curves.py @@ -6,7 +6,7 @@ import pandas as pd import numpy as np import pytest -from pandas.util.testing import assert_frame_equal +from pandas.testing import assert_frame_equal from windpowerlib.power_curves import ( smooth_power_curve, diff --git a/tests/test_power_output.py b/tests/test_power_output.py index d8fef326..b9dd72f7 100644 --- a/tests/test_power_output.py +++ b/tests/test_power_output.py @@ -8,7 +8,7 @@ import pandas as pd import pytest from numpy.testing import assert_allclose -from pandas.util.testing import assert_series_equal +from pandas.testing import assert_series_equal from windpowerlib.power_output import ( power_coefficient_curve, power_curve, diff --git a/tests/test_temperature.py b/tests/test_temperature.py index d0e2ba4b..3b86bc58 100644 --- a/tests/test_temperature.py +++ b/tests/test_temperature.py @@ -5,7 +5,7 @@ import pandas as pd import numpy as np -from pandas.util.testing import assert_series_equal +from pandas.testing import assert_series_equal from numpy.testing import assert_array_equal from windpowerlib.temperature import linear_gradient diff --git a/tests/test_tools.py b/tests/test_tools.py index 2392d89d..308e8580 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -4,7 +4,7 @@ """ import pandas as pd -from pandas.util.testing import assert_series_equal +from pandas.testing import assert_series_equal from windpowerlib.tools import ( linear_interpolation_extrapolation, diff --git a/tests/test_turbine_cluster_modelchain.py b/tests/test_turbine_cluster_modelchain.py index 463328b5..2aa8c5f1 100644 --- a/tests/test_turbine_cluster_modelchain.py +++ b/tests/test_turbine_cluster_modelchain.py @@ -6,7 +6,7 @@ import pytest import pandas as pd import numpy as np -from pandas.util.testing import assert_series_equal +from pandas.testing import assert_series_equal import windpowerlib.wind_farm as wf import windpowerlib.wind_turbine as wt diff --git a/tests/test_wake_losses.py b/tests/test_wake_losses.py index d27fd4a1..1d02d128 100644 --- a/tests/test_wake_losses.py +++ b/tests/test_wake_losses.py @@ -6,7 +6,7 @@ import pandas as pd import numpy as np import pytest -from pandas.util.testing import assert_series_equal +from pandas.testing import assert_series_equal from windpowerlib.wake_losses import ( reduce_wind_speed, diff --git a/tests/test_wind_speed.py b/tests/test_wind_speed.py index f4af4299..f92dff99 100644 --- a/tests/test_wind_speed.py +++ b/tests/test_wind_speed.py @@ -6,7 +6,7 @@ import pandas as pd import numpy as np import pytest -from pandas.util.testing import assert_series_equal +from pandas.testing import assert_series_equal from numpy.testing import assert_allclose from windpowerlib.wind_speed import logarithmic_profile, hellman From 710847b8aa815225934fc3e532b5f549664a0d12 Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 8 Jan 2024 15:00:23 -0800 Subject: [PATCH 02/42] Only add curve data with unique wind speeds Duplicated wind speed entries lead to errors --- windpowerlib/data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/windpowerlib/data.py b/windpowerlib/data.py index 309bcd7d..528adf79 100644 --- a/windpowerlib/data.py +++ b/windpowerlib/data.py @@ -216,9 +216,10 @@ def store_turbine_data_from_oedb( } ) ) - curves_df = pd.merge( - left=curves_df, right=df, how="outer", on="wind_speed" - ) + if not df.wind_speed.duplicated().any(): + curves_df = pd.merge( + left=curves_df, right=df, how="outer", on="wind_speed" + ) curves_df = curves_df.set_index("wind_speed").sort_index().transpose() # power curve values in W if curve_type == "power_curve": From ca9e4649476fd5cf1bdbc7bce40210e8b1c2c0ab Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 8 Jan 2024 15:04:20 -0800 Subject: [PATCH 03/42] Series.append is deprecated, use pd.concat instead --- windpowerlib/power_curves.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/windpowerlib/power_curves.py b/windpowerlib/power_curves.py index 43df0071..40238368 100644 --- a/windpowerlib/power_curves.py +++ b/windpowerlib/power_curves.py @@ -140,18 +140,26 @@ def smooth_power_curve( # Append wind speeds to `power_curve_wind_speeds` maximum_value = power_curve_wind_speeds.iloc[-1] + wind_speed_range while power_curve_wind_speeds.values[-1] < maximum_value: - power_curve_wind_speeds = power_curve_wind_speeds.append( - pd.Series( - power_curve_wind_speeds.iloc[-1] - + ( - power_curve_wind_speeds.iloc[5] - - power_curve_wind_speeds.iloc[4] - ), - index=[power_curve_wind_speeds.index[-1] + 1], - ) + power_curve_wind_speeds = pd.concat( + [ + power_curve_wind_speeds, + pd.Series( + power_curve_wind_speeds.iloc[-1] + + ( + power_curve_wind_speeds.iloc[5] + - power_curve_wind_speeds.iloc[4] + ), + index=[power_curve_wind_speeds.index[-1] + 1], + ) + ], + sort=True, ) - power_curve_values = power_curve_values.append( - pd.Series(0.0, index=[power_curve_values.index[-1] + 1]) + power_curve_values = pd.concat( + [ + power_curve_values, + pd.Series(0.0, index=[power_curve_values.index[-1] + 1]) + ], + sort=True, ) for power_curve_wind_speed in power_curve_wind_speeds: # Create array of wind speeds for the sum From e963667534e59afb7e555eaf6894c92e30db7bd3 Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 8 Jan 2024 15:05:25 -0800 Subject: [PATCH 04/42] Fix error - float can only be applied to single element --- windpowerlib/wind_turbine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windpowerlib/wind_turbine.py b/windpowerlib/wind_turbine.py index 1efef699..66556666 100644 --- a/windpowerlib/wind_turbine.py +++ b/windpowerlib/wind_turbine.py @@ -170,9 +170,9 @@ def __init__( logging.debug(msg.format(self.turbine_type)) if self.nominal_power is None and turbine_data is not None: - self.nominal_power = float(turbine_data["nominal_power"]) + self.nominal_power = float(turbine_data["nominal_power"].iloc[0]) if self.rotor_diameter is None and turbine_data is not None: - self.rotor_diameter = float(turbine_data["rotor_diameter"]) + self.rotor_diameter = float(turbine_data["rotor_diameter"].iloc[0]) if self.rotor_diameter: if self.hub_height <= 0.5 * self.rotor_diameter: From 3ce26c78df0333004a8e179711e74d6c25ecba5a Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 8 Jan 2024 15:05:58 -0800 Subject: [PATCH 05/42] Make sure data is sorted, sort is now per default False --- windpowerlib/power_curves.py | 1 + windpowerlib/wind_farm.py | 1 + windpowerlib/wind_turbine_cluster.py | 1 + 3 files changed, 3 insertions(+) diff --git a/windpowerlib/power_curves.py b/windpowerlib/power_curves.py index 40238368..2a693e0f 100644 --- a/windpowerlib/power_curves.py +++ b/windpowerlib/power_curves.py @@ -257,6 +257,7 @@ def wake_losses_to_power_curve( wind_farm_efficiency.set_index("wind_speed"), ], axis=1, + sort=True, ) # Add column with reduced power (nan values of efficiency are # interpolated) diff --git a/windpowerlib/wind_farm.py b/windpowerlib/wind_farm.py index c20c704a..7c9b8f97 100644 --- a/windpowerlib/wind_farm.py +++ b/windpowerlib/wind_farm.py @@ -446,6 +446,7 @@ def assign_power_curve( ), ], axis=1, + sort=True, ) # Aggregate all power curves wind_farm_power_curve = pd.DataFrame( diff --git a/windpowerlib/wind_turbine_cluster.py b/windpowerlib/wind_turbine_cluster.py index 166ea64f..da540ac3 100644 --- a/windpowerlib/wind_turbine_cluster.py +++ b/windpowerlib/wind_turbine_cluster.py @@ -211,6 +211,7 @@ def assign_power_curve( ) ], axis=1, + sort=True ) # Sum up power curves cluster_power_curve = pd.DataFrame( From 3df5e99411a609ef20f0386756f3ca30b1384412 Mon Sep 17 00:00:00 2001 From: birgits Date: Mon, 8 Jan 2024 15:28:50 -0800 Subject: [PATCH 06/42] Change import of weather data due to deprecation of date_parser --- example/modelchain_example.ipynb | 3 ++- example/modelchain_example.py | 2 +- example/simple_example.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/example/modelchain_example.ipynb b/example/modelchain_example.ipynb index 696301b2..5afb2eb8 100644 --- a/example/modelchain_example.ipynb +++ b/example/modelchain_example.ipynb @@ -144,7 +144,8 @@ " file,\n", " index_col=0,\n", " header=[0, 1],\n", - " date_parser=lambda idx: pd.to_datetime(idx, utc=True))\n", + " )\n", + " weather_df.index = pd.to_datetime(weather_df.index, utc=True)\n", " \n", " # change time zone\n", " weather_df.index = weather_df.index.tz_convert(\n", diff --git a/example/modelchain_example.py b/example/modelchain_example.py index 5ed87d73..18e49165 100644 --- a/example/modelchain_example.py +++ b/example/modelchain_example.py @@ -85,8 +85,8 @@ def get_weather_data(filename="weather.csv", **kwargs): file, index_col=0, header=[0, 1], - date_parser=lambda idx: pd.to_datetime(idx, utc=True), ) + weather_df.index = pd.to_datetime(weather_df.index, utc=True) # change time zone weather_df.index = weather_df.index.tz_convert("Europe/Berlin") diff --git a/example/simple_example.py b/example/simple_example.py index d42dad06..0023fe1f 100644 --- a/example/simple_example.py +++ b/example/simple_example.py @@ -79,8 +79,8 @@ def get_weather_data(filename="weather.csv", **kwargs): file, index_col=0, header=[0, 1], - date_parser=lambda idx: pd.to_datetime(idx, utc=True), ) + weather_df.index = pd.to_datetime(weather_df.index, utc=True) # change time zone weather_df.index = weather_df.index.tz_convert("Europe/Berlin") From 87924886fb05f4103e87819dd34014242fa16e2a Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 10 Jan 2024 14:06:47 -0800 Subject: [PATCH 07/42] Fix full load hours I couldn't reproduce the full load hours previously set (1956.164053) even with older versions of python (checked down to 3.8) and pandas (checked down to 1.3.5), so I assume, that the value was incorrect. It was possibly not checked, as the example tests are not in the tests directory but the example directory. --- example/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/test_examples.py b/example/test_examples.py index 8be5e9d0..de76fe21 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -42,7 +42,7 @@ def test_turbine_cluster_modelchain_example_flh(self): ) tc_mc_e.calculate_power_output(weather, example_farm, example_cluster) assert_allclose( - 1956.164053, + 2004.84125, (example_farm.power_output.sum() / example_farm.nominal_power), 0.01, ) From 4efa95c35f4185dc0c9463e1ec19c2609e3f7eb9 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 10 Jan 2024 14:50:56 -0800 Subject: [PATCH 08/42] Add github workflow for continuous testing --- .github/workflows/tests-coverage.yml | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/tests-coverage.yml diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml new file mode 100644 index 00000000..7647d663 --- /dev/null +++ b/.github/workflows/tests-coverage.yml @@ -0,0 +1,69 @@ +# Tests with pytest the package and monitors the coverage and sends it to coveralls.io +# Coverage is only send to coveralls.io when no pytest tests fail +name: "Tests & Coverage" + +on: [push] + +# Cancel jobs on new push +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + name: "${{ matrix.name-suffix }} at py${{ matrix.python-version }} on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: + - name-suffix: "coverage" + os: ubuntu-latest + python-version: 3.11 + - name-suffix: "basic" + os: ubuntu-latest + python-version: 3.10 + - name-suffix: "basic" + os: ubuntu-latest + python-version: 3.12 + - name-suffix: "basic" + os: windows-latest + python-version: 3.11 + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies (Linux) + if: runner.os == 'Linux' + run: | + python -m pip install --upgrade pip wheel setuptools + pip install .[dev] + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + uses: conda-incubator/setup-miniconda@v2 + run: + conda create -n testenv python=${{ matrix.python }} + conda activate testenv + python -m pip install .[dev] + + - name: Run tests + if: ${{ !(runner.os == 'Linux' && matrix.python-version == 3.9 && matrix.name-suffix == 'coverage') }} + run: | + python -m pytest --disable-warnings --color=yes -v + + - name: Run tests, coverage and send to coveralls + if: runner.os == 'Linux' && matrix.python-version == 3.9 && matrix.name-suffix == 'coverage' + run: | + coverage run --source=windpowerlib -m pytest --disable-warnings --color=yes -v + coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_SERVICE_NAME: github From 32fece2c92ed991ed703fd15328095d8636bafe0 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 10 Jan 2024 14:51:14 -0800 Subject: [PATCH 09/42] Update tested and supported python versions --- .travis.yml | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f515fb0..9b4e348c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,9 @@ language: python matrix: include: - - python: 3.6 - - python: 3.7 - - python: 3.8 + - python: 3.10 + - python: 3.11 + - python: 3.12 # command to install dependencies #before_install: diff --git a/pyproject.toml b/pyproject.toml index 3dbb3802..a11e2514 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 79 -target-version = ['py36', 'py37', 'py38'] +target-version = ['py310', 'py311', 'py312'] include = '\.pyi?$' exclude = ''' /( From dafb8bc1204bac421644b321b25fbd51bade1fc5 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 10 Jan 2024 14:53:28 -0800 Subject: [PATCH 10/42] Bug fix - cannot have uses and run key --- .github/workflows/tests-coverage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index 7647d663..9b48cb4f 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -48,7 +48,6 @@ jobs: - name: Install dependencies (Windows) if: runner.os == 'Windows' - uses: conda-incubator/setup-miniconda@v2 run: conda create -n testenv python=${{ matrix.python }} conda activate testenv From 64cb2d308f414f29bac176a5478bd296615975db Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 10 Jan 2024 15:02:55 -0800 Subject: [PATCH 11/42] Make python versions strings so that python3.10 is recognised correctly --- .github/workflows/tests-coverage.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index 9b48cb4f..56cf1378 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -20,16 +20,16 @@ jobs: include: - name-suffix: "coverage" os: ubuntu-latest - python-version: 3.11 + python-version: "3.11" - name-suffix: "basic" os: ubuntu-latest - python-version: 3.10 + python-version: "3.10" - name-suffix: "basic" os: ubuntu-latest - python-version: 3.12 + python-version: "3.12" - name-suffix: "basic" os: windows-latest - python-version: 3.11 + python-version: "3.11" steps: - name: Checkout repo From ea7e8a6f1508cf6ae03e5e04578dc14bd02d00d9 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 10 Jan 2024 15:03:28 -0800 Subject: [PATCH 12/42] Fix missing conda installation --- .github/workflows/tests-coverage.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index 56cf1378..c6efca13 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -40,17 +40,17 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies (Linux) - if: runner.os == 'Linux' + - name: Set up Conda + if: runner.os == 'Windows' + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + python-version: ${{ matrix.python-version }} + activate-environment: testenv + + - name: Install dependencies run: | python -m pip install --upgrade pip wheel setuptools - pip install .[dev] - - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: - conda create -n testenv python=${{ matrix.python }} - conda activate testenv python -m pip install .[dev] - name: Run tests From 89c23c445a6219aabf6159949e227b06f98f23db Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 09:53:22 -0800 Subject: [PATCH 13/42] Fix import of modelchain_example - previous import fails when notebook is run using pytest_notebook --- example/turbine_cluster_modelchain_example.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/turbine_cluster_modelchain_example.ipynb b/example/turbine_cluster_modelchain_example.ipynb index 634744ab..c555ae49 100644 --- a/example/turbine_cluster_modelchain_example.ipynb +++ b/example/turbine_cluster_modelchain_example.ipynb @@ -39,7 +39,7 @@ "source": [ "import pandas as pd\n", "\n", - "import modelchain_example as mc_e\n", + "from example import modelchain_example as mc_e\n", "from windpowerlib import TurbineClusterModelChain, WindTurbineCluster, WindFarm\n", "\n", "import logging\n", From 07649a958fcf7756c5407b0633b18b180c1f234b Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 09:54:52 -0800 Subject: [PATCH 14/42] Use pytest_notebook to test whether notebooks run without errors --- example/test_examples.py | 53 ++++++++++------------------------------ setup.py | 2 +- 2 files changed, 14 insertions(+), 41 deletions(-) diff --git a/example/test_examples.py b/example/test_examples.py index de76fe21..30d76103 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -4,14 +4,10 @@ """ import os -import subprocess -import tempfile -import nbformat -import sys from example import modelchain_example as mc_e from example import turbine_cluster_modelchain_example as tc_mc_e from numpy.testing import assert_allclose -import pytest +import pytest_notebook class TestExamples: @@ -57,50 +53,27 @@ def test_turbine_cluster_modelchain_example_flh(self): def _notebook_run(self, path): """ - Execute a notebook via nbconvert and collect output. + Execute a notebook and collect output. Returns (parsed nb object, execution errors) """ - dirname, __ = os.path.split(path) - os.chdir(dirname) - with tempfile.NamedTemporaryFile(suffix=".ipynb") as fout: - args = [ - "jupyter", - "nbconvert", - "--to", - "notebook", - "--execute", - "--ExecutePreprocessor.timeout=60", - "--output", - fout.name, - path, - ] - subprocess.check_call(args) - - fout.seek(0) - nb = nbformat.read(fout, nbformat.current_nbformat) - - errors = [ - output - for cell in nb.cells - if "outputs" in cell - for output in cell["outputs"] - if output.output_type == "error" - ] - - return nb, errors + notebook = pytest_notebook.notebook.load_notebook(path=path) + result = pytest_notebook.execution.execute_notebook( + notebook, + with_coverage=False, + timeout=600, + ) + return result.exec_error - @pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6") def test_modelchain_example_ipynb(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - nb, errors = self._notebook_run( + errors = self._notebook_run( os.path.join(dir_path, "modelchain_example.ipynb") ) - assert errors == [] + assert errors is None - @pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6") def test_turbine_cluster_modelchain_example_ipynb(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - nb, errors = self._notebook_run( + errors = self._notebook_run( os.path.join(dir_path, "turbine_cluster_modelchain_example.ipynb") ) - assert errors == [] + assert errors is None diff --git a/setup.py b/setup.py index cc4216a9..4f4ed646 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,9 @@ def read(fname): "pytest", "jupyter", "sphinx_rtd_theme", - "nbformat", "numpy", "matplotlib", + "pytest-notebook", ] }, ) From 1f482991e73860a0ef83d95d2a2991ca05a94e19 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 09:57:02 -0800 Subject: [PATCH 15/42] Fix tests - tests fail on windows as dtype on one side is np.int32 and on the other np.int64 --- tests/test_modelchain.py | 6 +++++- tests/test_power_curves.py | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_modelchain.py b/tests/test_modelchain.py index a9b8f657..3f7269bf 100644 --- a/tests/test_modelchain.py +++ b/tests/test_modelchain.py @@ -102,7 +102,11 @@ def test_temperature_hub(self): np.array([100, 10]), ] temp_exp = pd.Series(data=[267, 268], name=100) - assert_series_equal(test_mc.temperature_hub(weather_df), temp_exp) + assert_series_equal( + test_mc.temperature_hub(weather_df), + temp_exp, + check_dtype=False, + ) def test_density_hub(self): # Test modelchain with density_model='barometric' diff --git a/tests/test_power_curves.py b/tests/test_power_curves.py index 0eed069d..4195d87e 100644 --- a/tests/test_power_curves.py +++ b/tests/test_power_curves.py @@ -53,7 +53,9 @@ def test_smooth_power_curve(self): ) smoothed_curve_exp.index = np.arange(5, 10, 1) assert_frame_equal( - smooth_power_curve(**parameters)[5:10], smoothed_curve_exp + smooth_power_curve(**parameters)[5:10], + smoothed_curve_exp, + check_dtype=False, ) # Test Staffel_Pfenninger method @@ -73,7 +75,9 @@ def test_smooth_power_curve(self): ) smoothed_curve_exp.index = np.arange(5, 10, 1) assert_frame_equal( - smooth_power_curve(**parameters)[5:10], smoothed_curve_exp + smooth_power_curve(**parameters)[5:10], + smoothed_curve_exp, + check_dtype=False, ) # Raise ValueError - misspelling From a86b50f40ce3a20a07f5e09c85e9a5de027af00e Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 09:57:55 -0800 Subject: [PATCH 16/42] Try fixing failing tests on windows - on windows dtype here is np.int64 and on linux np.int32 --- tests/test_modelchain.py | 2 +- tests/test_turbine_cluster_modelchain.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_modelchain.py b/tests/test_modelchain.py index 3f7269bf..483f0b41 100644 --- a/tests/test_modelchain.py +++ b/tests/test_modelchain.py @@ -456,7 +456,7 @@ def test_heigths_as_string(self): # Heights in the original DataFrame are of type np.int64 assert isinstance( - self.weather_df.columns.get_level_values(1)[0], np.int64 + self.weather_df.columns.get_level_values(1)[0], np.int_ ) assert isinstance(string_weather.columns.get_level_values(1)[0], str) diff --git a/tests/test_turbine_cluster_modelchain.py b/tests/test_turbine_cluster_modelchain.py index 2aa8c5f1..2e3342c2 100644 --- a/tests/test_turbine_cluster_modelchain.py +++ b/tests/test_turbine_cluster_modelchain.py @@ -373,7 +373,7 @@ def test_heigths_as_string(self): # Heights in the original DataFrame are of type np.int64 assert isinstance( - self.weather_df.columns.get_level_values(1)[0], np.int64 + self.weather_df.columns.get_level_values(1)[0], np.int_ ) assert isinstance(string_weather.columns.get_level_values(1)[0], str) From bec03ade11fc451ae60ec2ab0e975f761870a303 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 10:11:18 -0800 Subject: [PATCH 17/42] Try fixing different index dtype - on windows dtype is int32 and on linux int64 --- tests/test_power_curves.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_power_curves.py b/tests/test_power_curves.py index 4195d87e..0d83d882 100644 --- a/tests/test_power_curves.py +++ b/tests/test_power_curves.py @@ -56,6 +56,7 @@ def test_smooth_power_curve(self): smooth_power_curve(**parameters)[5:10], smoothed_curve_exp, check_dtype=False, + check_index_type=False, ) # Test Staffel_Pfenninger method From b051746c81fe001fcc11206acbe6a487d11a1501 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 10:12:57 -0800 Subject: [PATCH 18/42] Try fixing import error of example.modelchain_example when notebooks are run on github --- example/test_examples.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/example/test_examples.py b/example/test_examples.py index 30d76103..1cd74913 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -56,7 +56,9 @@ def _notebook_run(self, path): Execute a notebook and collect output. Returns (parsed nb object, execution errors) """ - notebook = pytest_notebook.notebook.load_notebook(path=path) + dirname, nb_name = os.path.split(path) + os.chdir(dirname) + notebook = pytest_notebook.notebook.load_notebook(path=nb_name) result = pytest_notebook.execution.execute_notebook( notebook, with_coverage=False, From 9aec02e989662014ecdf89ed2a3c4ed390d9a73f Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 10:43:28 -0800 Subject: [PATCH 19/42] Fix test for windows - on windows dtype on the left is int64 and on the right int32 --- tests/test_power_curves.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_power_curves.py b/tests/test_power_curves.py index 0d83d882..8ed59f7f 100644 --- a/tests/test_power_curves.py +++ b/tests/test_power_curves.py @@ -79,6 +79,7 @@ def test_smooth_power_curve(self): smooth_power_curve(**parameters)[5:10], smoothed_curve_exp, check_dtype=False, + check_index_type=False, ) # Raise ValueError - misspelling From cac318701a33cc17600c812180858d170fe370fa Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 11:30:37 -0800 Subject: [PATCH 20/42] Revert "Use pytest_notebook to test whether notebooks run without errors" This reverts commit 07649a958fcf7756c5407b0633b18b180c1f234b. --- example/test_examples.py | 53 +++++++++++++++++++++++++++++----------- setup.py | 2 +- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/example/test_examples.py b/example/test_examples.py index 1cd74913..de76fe21 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -4,10 +4,14 @@ """ import os +import subprocess +import tempfile +import nbformat +import sys from example import modelchain_example as mc_e from example import turbine_cluster_modelchain_example as tc_mc_e from numpy.testing import assert_allclose -import pytest_notebook +import pytest class TestExamples: @@ -53,29 +57,50 @@ def test_turbine_cluster_modelchain_example_flh(self): def _notebook_run(self, path): """ - Execute a notebook and collect output. + Execute a notebook via nbconvert and collect output. Returns (parsed nb object, execution errors) """ - dirname, nb_name = os.path.split(path) + dirname, __ = os.path.split(path) os.chdir(dirname) - notebook = pytest_notebook.notebook.load_notebook(path=nb_name) - result = pytest_notebook.execution.execute_notebook( - notebook, - with_coverage=False, - timeout=600, - ) - return result.exec_error + with tempfile.NamedTemporaryFile(suffix=".ipynb") as fout: + args = [ + "jupyter", + "nbconvert", + "--to", + "notebook", + "--execute", + "--ExecutePreprocessor.timeout=60", + "--output", + fout.name, + path, + ] + subprocess.check_call(args) + + fout.seek(0) + nb = nbformat.read(fout, nbformat.current_nbformat) + + errors = [ + output + for cell in nb.cells + if "outputs" in cell + for output in cell["outputs"] + if output.output_type == "error" + ] + + return nb, errors + @pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6") def test_modelchain_example_ipynb(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - errors = self._notebook_run( + nb, errors = self._notebook_run( os.path.join(dir_path, "modelchain_example.ipynb") ) - assert errors is None + assert errors == [] + @pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6") def test_turbine_cluster_modelchain_example_ipynb(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - errors = self._notebook_run( + nb, errors = self._notebook_run( os.path.join(dir_path, "turbine_cluster_modelchain_example.ipynb") ) - assert errors is None + assert errors == [] diff --git a/setup.py b/setup.py index 4f4ed646..cc4216a9 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,9 @@ def read(fname): "pytest", "jupyter", "sphinx_rtd_theme", + "nbformat", "numpy", "matplotlib", - "pytest-notebook", ] }, ) From 4dd58585ed09e55b52a40750911c412511945e2e Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 11:31:32 -0800 Subject: [PATCH 21/42] Increase timeout to avoid error when tests are run on github --- example/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/test_examples.py b/example/test_examples.py index de76fe21..f48c591c 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -69,7 +69,7 @@ def _notebook_run(self, path): "--to", "notebook", "--execute", - "--ExecutePreprocessor.timeout=60", + "--ExecutePreprocessor.timeout=600", "--output", fout.name, path, From 01be601145a1ce3ad9db3f3fa2c782293fe9c90e Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 11:31:53 -0800 Subject: [PATCH 22/42] Not needed anymore as lower python versions are not supported anymore --- example/test_examples.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/example/test_examples.py b/example/test_examples.py index f48c591c..0bca612c 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -89,7 +89,6 @@ def _notebook_run(self, path): return nb, errors - @pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6") def test_modelchain_example_ipynb(self): dir_path = os.path.dirname(os.path.realpath(__file__)) nb, errors = self._notebook_run( @@ -97,7 +96,6 @@ def test_modelchain_example_ipynb(self): ) assert errors == [] - @pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6") def test_turbine_cluster_modelchain_example_ipynb(self): dir_path = os.path.dirname(os.path.realpath(__file__)) nb, errors = self._notebook_run( From f26de07a0217612ca1c7646010c551b452b9edee Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 11:35:06 -0800 Subject: [PATCH 23/42] Change import back as tests fail now --- example/turbine_cluster_modelchain_example.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/turbine_cluster_modelchain_example.ipynb b/example/turbine_cluster_modelchain_example.ipynb index c555ae49..634744ab 100644 --- a/example/turbine_cluster_modelchain_example.ipynb +++ b/example/turbine_cluster_modelchain_example.ipynb @@ -39,7 +39,7 @@ "source": [ "import pandas as pd\n", "\n", - "from example import modelchain_example as mc_e\n", + "import modelchain_example as mc_e\n", "from windpowerlib import TurbineClusterModelChain, WindTurbineCluster, WindFarm\n", "\n", "import logging\n", From 5332a8ea06df717301154a1fe9e7fa5bfa518fc7 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 13:56:37 -0800 Subject: [PATCH 24/42] Try setting directory to fix error for windows PermissionError: [Errno 13] Permission denied --- example/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/test_examples.py b/example/test_examples.py index 0bca612c..68903191 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -62,7 +62,7 @@ def _notebook_run(self, path): """ dirname, __ = os.path.split(path) os.chdir(dirname) - with tempfile.NamedTemporaryFile(suffix=".ipynb") as fout: + with tempfile.NamedTemporaryFile(suffix=".ipynb", dir=dirname) as fout: args = [ "jupyter", "nbconvert", From f2755476a3a8ef52eae9dc2003e4e2f9c3ca4478 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 14:30:27 -0800 Subject: [PATCH 25/42] Try setting permissions to fix PermissionError --- .github/workflows/tests-coverage.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index c6efca13..cf2d7738 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -53,6 +53,11 @@ jobs: python -m pip install --upgrade pip wheel setuptools python -m pip install .[dev] + - name: Fix permission + if: runner.os == 'Windows' + run: | + sudo chmod -R 777 $GITHUB_WORKSPACE/public + - name: Run tests if: ${{ !(runner.os == 'Linux' && matrix.python-version == 3.9 && matrix.name-suffix == 'coverage') }} run: | From 1d7055ff6df0650f5fd6b431b1e6751e6906fb0c Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 14:31:23 -0800 Subject: [PATCH 26/42] Comment other tests to run only windows tests --- .github/workflows/tests-coverage.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index cf2d7738..752ef7c3 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -18,15 +18,15 @@ jobs: fail-fast: false matrix: include: - - name-suffix: "coverage" - os: ubuntu-latest - python-version: "3.11" - - name-suffix: "basic" - os: ubuntu-latest - python-version: "3.10" - - name-suffix: "basic" - os: ubuntu-latest - python-version: "3.12" +# - name-suffix: "coverage" +# os: ubuntu-latest +# python-version: "3.11" +# - name-suffix: "basic" +# os: ubuntu-latest +# python-version: "3.10" +# - name-suffix: "basic" +# os: ubuntu-latest +# python-version: "3.12" - name-suffix: "basic" os: windows-latest python-version: "3.11" From b668b70ce2d4a750e070675fe29b21444175c1ea Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 15:04:28 -0800 Subject: [PATCH 27/42] Change permission --- .github/workflows/tests-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index 752ef7c3..e7f3c8cb 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -56,7 +56,7 @@ jobs: - name: Fix permission if: runner.os == 'Windows' run: | - sudo chmod -R 777 $GITHUB_WORKSPACE/public + icacls $GITHUB_WORKSPACE /grant:w User:F - name: Run tests if: ${{ !(runner.os == 'Linux' && matrix.python-version == 3.9 && matrix.name-suffix == 'coverage') }} From d45d448d5c8bf719b4679afbc896352e65e0a920 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 15:23:33 -0800 Subject: [PATCH 28/42] Remove setting permissions again as it did not work correctly --- .github/workflows/tests-coverage.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index e7f3c8cb..89c6ca34 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -53,11 +53,6 @@ jobs: python -m pip install --upgrade pip wheel setuptools python -m pip install .[dev] - - name: Fix permission - if: runner.os == 'Windows' - run: | - icacls $GITHUB_WORKSPACE /grant:w User:F - - name: Run tests if: ${{ !(runner.os == 'Linux' && matrix.python-version == 3.9 && matrix.name-suffix == 'coverage') }} run: | From 028aa8d2fa9ddf87981dc2a20cc558e23582c478 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 15:27:09 -0800 Subject: [PATCH 29/42] Try workaround - don't import functions from external python file --- .../turbine_cluster_modelchain_example.ipynb | 113 ++++++++++++++++-- 1 file changed, 100 insertions(+), 13 deletions(-) diff --git a/example/turbine_cluster_modelchain_example.ipynb b/example/turbine_cluster_modelchain_example.ipynb index 634744ab..04fe8b17 100644 --- a/example/turbine_cluster_modelchain_example.ipynb +++ b/example/turbine_cluster_modelchain_example.ipynb @@ -28,7 +28,9 @@ "source": [ "### Imports and initialization of wind turbines\n", "\n", - "The import of weather data and the initialization of wind turbines is done as in the ``modelchain_example``. Be aware that currently for wind farm and wind cluster calculations wind turbines need to have a power curve as some calculations do not work with the power coefficient curve." + "The import of weather data and the initialization of wind turbines are taken from the ``modelchain_example``. See there for more information.\n", + "\n", + "Also, be aware that currently for wind farm and wind cluster calculations wind turbines need to have a power curve as some calculations do not work with the power coefficient curve." ] }, { @@ -37,10 +39,10 @@ "metadata": {}, "outputs": [], "source": [ + "import os\n", "import pandas as pd\n", "\n", - "import modelchain_example as mc_e\n", - "from windpowerlib import TurbineClusterModelChain, WindTurbineCluster, WindFarm\n", + "from windpowerlib import create_power_curve, TurbineClusterModelChain, WindFarm, WindTurbine, WindTurbineCluster\n", "\n", "import logging\n", "logging.getLogger().setLevel(logging.DEBUG)" @@ -59,21 +61,106 @@ "height 10 80 2 10 0\n", "2010-01-01 00:00:00+01:00 5.32697 7.80697 267.60 267.57 98405.7\n", "2010-01-01 01:00:00+01:00 5.46199 7.86199 267.60 267.55 98382.7\n", - "2010-01-01 02:00:00+01:00 5.67899 8.59899 267.61 267.54 98362.9\n", - "\n", - "nominal power of my_turbine: 3000000.0\n" + "2010-01-01 02:00:00+01:00 5.67899 8.59899 267.61 267.54 98362.9\n" + ] + } + ], + "source": [ + "def get_weather_data(filename='weather.csv', **kwargs):\n", + " r\"\"\"\n", + " Imports weather data from a file.\n", + "\n", + " \"\"\"\n", + "\n", + " if 'datapath' not in kwargs:\n", + " kwargs['datapath'] = os.path.dirname(__file__)\n", + " \n", + " file = os.path.join(kwargs['datapath'], filename)\n", + " \n", + " # download example weather data file in case it does not yet exist\n", + " if not os.path.isfile(file):\n", + " logging.debug(\"Download weather data for example.\")\n", + " req = requests.get(\"https://osf.io/59bqn/download\")\n", + " with open(file, \"wb\") as fout:\n", + " fout.write(req.content)\n", + " \n", + " # read csv file \n", + " weather_df = pd.read_csv(\n", + " file,\n", + " index_col=0,\n", + " header=[0, 1],\n", + " )\n", + " weather_df.index = pd.to_datetime(weather_df.index, utc=True)\n", + " \n", + " # change time zone\n", + " weather_df.index = weather_df.index.tz_convert(\n", + " 'Europe/Berlin')\n", + " \n", + " return weather_df\n", + "\n", + "# Read weather data from csv\n", + "weather = get_weather_data(filename='weather.csv', datapath='')\n", + "print(weather[['wind_speed', 'temperature', 'pressure']][0:3])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nominal power of my_turbine: 3000000.0\n" ] } ], "source": [ - "# Get weather data\n", - "weather = mc_e.get_weather_data('weather.csv')\n", - "print(weather[['wind_speed', 'temperature', 'pressure']][0:3])\n", + "def initialize_wind_turbines():\n", + " r\"\"\"\n", + " Initializes three WindTurbine objects.\n", + "\n", + " \"\"\"\n", + " enercon_e126 = {\n", + " \"turbine_type\": \"E-126/4200\", # turbine type as in register\n", + " \"hub_height\": 135, # in m\n", + " }\n", + " e126 = WindTurbine(**enercon_e126)\n", + "\n", + " my_turbine = {\n", + " \"nominal_power\": 3e6, # in W\n", + " \"hub_height\": 105, # in m\n", + " \"power_curve\": pd.DataFrame(\n", + " data={\n", + " \"value\": [\n", + " p * 1000\n", + " for p in [0.0, 26.0, 180.0, 1500.0, 3000.0, 3000.0]\n", + " ], # in W\n", + " \"wind_speed\": [0.0, 3.0, 5.0, 10.0, 15.0, 25.0],\n", + " }\n", + " ), # in m/s\n", + " }\n", + " my_turbine = WindTurbine(**my_turbine)\n", + "\n", + " my_power = pd.Series(\n", + " [0.0, 39000.0, 270000.0, 2250000.0, 4500000.0, 4500000.0]\n", + " )\n", + " my_wind_speed = (0.0, 3.0, 5.0, 10.0, 15.0, 25.0)\n", + "\n", + " my_turbine2 = {\n", + " \"nominal_power\": 6e6, # in W\n", + " \"hub_height\": 115, # in m\n", + " \"power_curve\": create_power_curve(\n", + " wind_speed=my_wind_speed, power=my_power\n", + " ),\n", + " }\n", + " my_turbine2 = WindTurbine(**my_turbine2)\n", + "\n", + " return my_turbine, e126, my_turbine2\n", "\n", - "# Initialize wind turbines\n", - "my_turbine, e126, my_turbine2 = mc_e.initialize_wind_turbines()\n", - "print()\n", - "print('nominal power of my_turbine: {}'.format(my_turbine.nominal_power))" + "my_turbine, e126, my_turbine2 = initialize_wind_turbines()\n", + "print('Nominal power of my_turbine: {}'.format(my_turbine.nominal_power))" ] }, { From 4e9fd2cb1f13c5c39a6207ec572c7c0afd0039d6 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 15:27:19 -0800 Subject: [PATCH 30/42] Remove old imports --- example/test_examples.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/example/test_examples.py b/example/test_examples.py index 68903191..1f0947fc 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -7,11 +7,9 @@ import subprocess import tempfile import nbformat -import sys from example import modelchain_example as mc_e from example import turbine_cluster_modelchain_example as tc_mc_e from numpy.testing import assert_allclose -import pytest class TestExamples: From edc686284891381b0c000efe8d8eeb54779158a2 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 15:27:40 -0800 Subject: [PATCH 31/42] Undo commenting linux tests --- .github/workflows/tests-coverage.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests-coverage.yml b/.github/workflows/tests-coverage.yml index 89c6ca34..c6efca13 100644 --- a/.github/workflows/tests-coverage.yml +++ b/.github/workflows/tests-coverage.yml @@ -18,15 +18,15 @@ jobs: fail-fast: false matrix: include: -# - name-suffix: "coverage" -# os: ubuntu-latest -# python-version: "3.11" -# - name-suffix: "basic" -# os: ubuntu-latest -# python-version: "3.10" -# - name-suffix: "basic" -# os: ubuntu-latest -# python-version: "3.12" + - name-suffix: "coverage" + os: ubuntu-latest + python-version: "3.11" + - name-suffix: "basic" + os: ubuntu-latest + python-version: "3.10" + - name-suffix: "basic" + os: ubuntu-latest + python-version: "3.12" - name-suffix: "basic" os: windows-latest python-version: "3.11" From 5f418bcf5c0dea9dd343fb1974e03cbf99a77c35 Mon Sep 17 00:00:00 2001 From: birgits Date: Fri, 12 Jan 2024 15:47:09 -0800 Subject: [PATCH 32/42] Use pytest-notebook again to avoid Permission denied error on windows --- example/test_examples.py | 51 +++++-------------- .../turbine_cluster_modelchain_example.ipynb | 1 + setup.py | 4 +- 3 files changed, 17 insertions(+), 39 deletions(-) diff --git a/example/test_examples.py b/example/test_examples.py index 1f0947fc..ea492f9c 100644 --- a/example/test_examples.py +++ b/example/test_examples.py @@ -4,12 +4,10 @@ """ import os -import subprocess -import tempfile -import nbformat from example import modelchain_example as mc_e from example import turbine_cluster_modelchain_example as tc_mc_e from numpy.testing import assert_allclose +import pytest_notebook class TestExamples: @@ -55,48 +53,27 @@ def test_turbine_cluster_modelchain_example_flh(self): def _notebook_run(self, path): """ - Execute a notebook via nbconvert and collect output. - Returns (parsed nb object, execution errors) + Execute a notebook and collect output. + Returns execution errors. """ - dirname, __ = os.path.split(path) - os.chdir(dirname) - with tempfile.NamedTemporaryFile(suffix=".ipynb", dir=dirname) as fout: - args = [ - "jupyter", - "nbconvert", - "--to", - "notebook", - "--execute", - "--ExecutePreprocessor.timeout=600", - "--output", - fout.name, - path, - ] - subprocess.check_call(args) - - fout.seek(0) - nb = nbformat.read(fout, nbformat.current_nbformat) - - errors = [ - output - for cell in nb.cells - if "outputs" in cell - for output in cell["outputs"] - if output.output_type == "error" - ] - - return nb, errors + notebook = pytest_notebook.notebook.load_notebook(path=path) + result = pytest_notebook.execution.execute_notebook( + notebook, + with_coverage=False, + timeout=600, + ) + return result.exec_error def test_modelchain_example_ipynb(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - nb, errors = self._notebook_run( + errors = self._notebook_run( os.path.join(dir_path, "modelchain_example.ipynb") ) - assert errors == [] + assert errors is None def test_turbine_cluster_modelchain_example_ipynb(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - nb, errors = self._notebook_run( + errors = self._notebook_run( os.path.join(dir_path, "turbine_cluster_modelchain_example.ipynb") ) - assert errors == [] + assert errors is None diff --git a/example/turbine_cluster_modelchain_example.ipynb b/example/turbine_cluster_modelchain_example.ipynb index 04fe8b17..a512d580 100644 --- a/example/turbine_cluster_modelchain_example.ipynb +++ b/example/turbine_cluster_modelchain_example.ipynb @@ -41,6 +41,7 @@ "source": [ "import os\n", "import pandas as pd\n", + "import requests\n", "\n", "from windpowerlib import create_power_curve, TurbineClusterModelChain, WindFarm, WindTurbine, WindTurbineCluster\n", "\n", diff --git a/setup.py b/setup.py index cc4216a9..3aed701e 100644 --- a/setup.py +++ b/setup.py @@ -24,15 +24,15 @@ def read(fname): long_description=read("README.rst"), long_description_content_type="text/x-rst", zip_safe=False, - install_requires=["pandas >= 0.20.0", "requests"], + install_requires=["pandas >= 0.20.0", "requests"], # check extras_require={ "dev": [ "pytest", "jupyter", "sphinx_rtd_theme", - "nbformat", "numpy", "matplotlib", + "pytest-notebook", ] }, ) From 8d488a9c8e4edadea9e10b9d8425a6f0082b9dbf Mon Sep 17 00:00:00 2001 From: birgits Date: Tue, 16 Jan 2024 12:12:45 -0800 Subject: [PATCH 33/42] Update documentation --- README.rst | 4 ++-- doc/getting_started.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index fad1a1bb..a0cad423 100644 --- a/README.rst +++ b/README.rst @@ -33,13 +33,13 @@ Go to the `download page = 3.6) environment, use pypi to install the latest windpowerlib version: +If you have a working Python 3 environment, use pypi to install the latest windpowerlib version: :: pip install windpowerlib -The windpowerlib is designed for Python 3 and tested on Python >= 3.5. We highly recommend to use virtual environments. +The windpowerlib is designed for Python 3 and tested on Python >= 3.10. We highly recommend to use virtual environments. Please see the `installation page `_ of the oemof documentation for complete instructions on how to install python and a virtual environment on your operating system. Optional Packages diff --git a/doc/getting_started.rst b/doc/getting_started.rst index d2a0d83b..169f3360 100644 --- a/doc/getting_started.rst +++ b/doc/getting_started.rst @@ -43,7 +43,7 @@ If you have a working Python 3 environment, use pypi to install the latest windp pip install windpowerlib -The windpowerlib is designed for Python 3 and tested on Python >= 3.5. We highly recommend to use virtual environments. +The windpowerlib is designed for Python 3 and tested on Python >= 3.10. We highly recommend to use virtual environments. Please see the `installation page `_ of the oemof documentation for complete instructions on how to install python and a virtual environment on your operating system. Optional Packages From f5f3d8f61b52b2dd999f5593a01b4135316297fc Mon Sep 17 00:00:00 2001 From: birgits Date: Tue, 16 Jan 2024 12:12:54 -0800 Subject: [PATCH 34/42] Add changes to whatsnew --- doc/whatsnew/v0-2-2.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/whatsnew/v0-2-2.rst b/doc/whatsnew/v0-2-2.rst index 89551de5..be840c13 100644 --- a/doc/whatsnew/v0-2-2.rst +++ b/doc/whatsnew/v0-2-2.rst @@ -1,6 +1,11 @@ v0.2.2 () ++++++++++++++++++++++++++++++ +* Updated the code basis to work for newer versions of python (support for python 3.6 to + python 3.9 is discontinued, supported python versions are now >= python 3.9) and added + github actions to run tests automatically when changes are pushed to github + (`PR 136 `_). + Contributors ############ - * Author \ No newline at end of file + * Birgit Schachler \ No newline at end of file From d274ce60e40608fbb6f1a96406bbe596469cf72f Mon Sep 17 00:00:00 2001 From: birgits Date: Tue, 16 Jan 2024 12:13:14 -0800 Subject: [PATCH 35/42] Remove lower limit for pandas as it is not needed anymore --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3aed701e..a9fb4e58 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def read(fname): long_description=read("README.rst"), long_description_content_type="text/x-rst", zip_safe=False, - install_requires=["pandas >= 0.20.0", "requests"], # check + install_requires=["pandas", "requests"], extras_require={ "dev": [ "pytest", From 296e58e3efb81c2a18f42c1a5ad8844e813b148b Mon Sep 17 00:00:00 2001 From: birgits Date: Tue, 30 Jan 2024 15:54:07 -0800 Subject: [PATCH 36/42] Handle faulty turbine library data --- tests/test_data_handling.py | 9 +++- windpowerlib/data.py | 102 +++++++++++++++++++++++++----------- 2 files changed, 77 insertions(+), 34 deletions(-) diff --git a/tests/test_data_handling.py b/tests/test_data_handling.py index bf31bf37..d5ea93e3 100644 --- a/tests/test_data_handling.py +++ b/tests/test_data_handling.py @@ -4,6 +4,7 @@ """ import filecmp +import logging import os from shutil import copyfile @@ -85,14 +86,18 @@ def test_get_turbine_types(self, capsys): with pytest.raises(ValueError, match=msg): get_turbine_types("wrong") - def test_store_turbine_data_from_oedb(self): + def test_store_turbine_data_from_oedb(self, caplog): """Test `store_turbine_data_from_oedb` function.""" t = {} for fn in os.listdir(self.orig_path): t[fn] = os.path.getmtime(os.path.join(self.orig_path, fn)) - store_turbine_data_from_oedb() + with caplog.at_level(logging.WARNING): + store_turbine_data_from_oedb() for fn in os.listdir(self.orig_path): assert t[fn] < os.path.getmtime(os.path.join(self.orig_path, fn)) + assert "The turbine library data contains too many faulty" not in caplog.text + assert "No cp-curve but has_cp_curve=True" not in caplog.text + assert "No power curve but has_power_curve=True" not in caplog.text def test_wrong_url_load_turbine_data(self): """Load turbine data from oedb with a wrong schema.""" diff --git a/windpowerlib/data.py b/windpowerlib/data.py index 528adf79..9d9cc1ff 100644 --- a/windpowerlib/data.py +++ b/windpowerlib/data.py @@ -187,46 +187,84 @@ def store_turbine_data_from_oedb( # get all power (coefficient) curves and save them to file for curve_type in ["power_curve", "power_coefficient_curve"]: + broken_turbine_data = [] curves_df = pd.DataFrame(columns=["wind_speed"]) for index in turbine_data.index: if ( turbine_data["{}_wind_speeds".format(curve_type)][index] and turbine_data["{}_values".format(curve_type)][index] ): - df = ( - pd.DataFrame( - data=[ - eval( - turbine_data[ - "{}_wind_speeds".format(curve_type) - ][index] - ), - eval( - turbine_data["{}_values".format(curve_type)][ - index - ] - ), - ] - ) - .transpose() - .rename( - columns={ - 0: "wind_speed", - 1: turbine_data["turbine_type"][index], - } + try: + df = ( + pd.DataFrame( + data=[ + eval( + turbine_data[ + "{}_wind_speeds".format(curve_type) + ][index] + ), + eval( + turbine_data["{}_values".format(curve_type)][ + index + ] + ), + ] + ) + .transpose() + .rename( + columns={ + 0: "wind_speed", + 1: turbine_data["turbine_type"][index], + } + ) ) + if not df.wind_speed.duplicated().any(): + curves_df = pd.merge( + left=curves_df, right=df, how="outer", on="wind_speed" + ) + except: + broken_turbine_data.append(turbine_data.loc[index, "turbine_type"]) + + # warning in case of broken turbine data + if len(broken_turbine_data) > 0: + issue_link = ("https://github.com/OpenEnergyPlatform/data-preprocessing" + "/issues/28") + # in case only some data is faulty, only give out warning + if len(broken_turbine_data) < 0.2 * len(turbine_data): + logging.warning( + f"The turbine library data contains faulty {curve_type}s. The " + f"{curve_type} data can therefore not be loaded for the following " + f"turbines: {broken_turbine_data}. " + f"Please report this in the following issue, in case it hasn't " + f"already been reported: {issue_link}" ) - if not df.wind_speed.duplicated().any(): - curves_df = pd.merge( - left=curves_df, right=df, how="outer", on="wind_speed" - ) - curves_df = curves_df.set_index("wind_speed").sort_index().transpose() - # power curve values in W - if curve_type == "power_curve": - curves_df *= 1000 - curves_df.index.name = "turbine_type" - curves_df.sort_index(inplace=True) - curves_df.to_csv(filename.format("{}s".format(curve_type))) + save_turbine_data = True + # set has_power_(coefficient)_curve to False for faulty turbines + for turb in broken_turbine_data: + ind = turbine_data[turbine_data.turbine_type == turb].index[0] + col = ("has_power_curve" if curve_type == "power_curve" + else "has_cp_curve") + turbine_data.at[ind, col] = False + # in case most data is faulty, do not store downloaded data + else: + logging.warning( + f"The turbine library data contains too many faulty {curve_type}s," + f"wherefore {curve_type} data is not loaded from the oedb. " + f"Please report this in the following issue, in case it hasn't " + f"already been reported: {issue_link}" + ) + save_turbine_data = False + else: + save_turbine_data = True + + if save_turbine_data: + curves_df = curves_df.set_index("wind_speed").sort_index().transpose() + # power curve values in W + if curve_type == "power_curve": + curves_df *= 1000 + curves_df.index.name = "turbine_type" + curves_df.sort_index(inplace=True) + curves_df.to_csv(filename.format("{}s".format(curve_type))) # get turbine data and save to file (excl. curves) turbine_data_df = turbine_data.drop( From 8a9a7695905d2572606e29a49b62e2b295b69072 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 31 Jan 2024 17:54:19 -0800 Subject: [PATCH 37/42] Add readthedocs.yml --- .readthedocs.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..edd0a241 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,28 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: doc/conf.py + +# Build documentation with MkDocs +#mkdocs: +# configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +# Optionally set the version of Python and requirements required to build your docs +python: + install: + - requirements: doc/requirements.txt + +# Set the version of Python +build: + os: ubuntu-22.04 + tools: + python: "3.11" From 649fca044fab23f2a74433300a6d28cf1e7fbec4 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 31 Jan 2024 18:03:42 -0800 Subject: [PATCH 38/42] Add sphinx_rtd_theme to rtd requirements --- doc/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/requirements.txt b/doc/requirements.txt index a8f73dd3..05a03dfa 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,5 @@ sphinx>=1.4 +sphinx_rtd_theme ipykernel nbsphinx pandas From 285a96a4f01d5bde02fd44e2e102790539403772 Mon Sep 17 00:00:00 2001 From: birgits Date: Wed, 31 Jan 2024 18:08:40 -0800 Subject: [PATCH 39/42] Remove asterisk --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 438f4259..83f8cca3 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def read(fname): packages=["windpowerlib"], package_data={ "windpowerlib": [ - os.path.join("data", "**.csv"), + os.path.join("data", "*.csv"), os.path.join("data", "default_turbine_data", "*.csv"), os.path.join("oedb", "*.csv"), ] From 3a6ed88234de39cbee6cc7fb4e5c16d3b2ffecf9 Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 1 Feb 2024 10:20:25 -0800 Subject: [PATCH 40/42] Fix failing docstring test for NaN values in "has_cp_curve" --- windpowerlib/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windpowerlib/data.py b/windpowerlib/data.py index 9d9cc1ff..718e4ee3 100644 --- a/windpowerlib/data.py +++ b/windpowerlib/data.py @@ -92,10 +92,10 @@ def get_turbine_types(turbine_library="local", print_out=True, filter_=True): + "but must be 'local' or 'oedb'." ) if filter_: - cp_curves_df = df.loc[df["has_cp_curve"]][ + cp_curves_df = df.loc[df["has_cp_curve"].fillna(False)][ ["manufacturer", "turbine_type", "has_cp_curve"] ] - p_curves_df = df.loc[df["has_power_curve"]][ + p_curves_df = df.loc[df["has_power_curve"].fillna(False)][ ["manufacturer", "turbine_type", "has_power_curve"] ] curves_df = pd.merge( From d57801ed395944e83cfa5277da28d013a69fc340 Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 1 Feb 2024 10:20:56 -0800 Subject: [PATCH 41/42] Adapt test so that "oedb" is checked as well ("local" is checked in docstring test) --- tests/test_data_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data_handling.py b/tests/test_data_handling.py index d5ea93e3..e54ab044 100644 --- a/tests/test_data_handling.py +++ b/tests/test_data_handling.py @@ -78,7 +78,7 @@ def test_broken_pwr_curve(self): def test_get_turbine_types(self, capsys): """Test the `get_turbine_types` function.""" - get_turbine_types() + get_turbine_types(turbine_library="oedb") captured = capsys.readouterr() assert "Enercon" in captured.out get_turbine_types("oedb", print_out=False, filter_=False) From b4db824addba5e5dbb48942eaf5af0f7871d9882 Mon Sep 17 00:00:00 2001 From: birgits Date: Thu, 8 Feb 2024 17:38:59 -0800 Subject: [PATCH 42/42] Move extracting curve data from oedb turbine library to own function to allow testing independent from oedb data --- tests/test_data_handling.py | 46 +++++++++++ windpowerlib/data.py | 159 ++++++++++++++++++++++++------------ 2 files changed, 154 insertions(+), 51 deletions(-) diff --git a/tests/test_data_handling.py b/tests/test_data_handling.py index e54ab044..b53d2bca 100644 --- a/tests/test_data_handling.py +++ b/tests/test_data_handling.py @@ -17,6 +17,7 @@ get_turbine_types, restore_default_turbine_data, store_turbine_data_from_oedb, + _process_and_save_oedb_data, ) @@ -99,6 +100,51 @@ def test_store_turbine_data_from_oedb(self, caplog): assert "No cp-curve but has_cp_curve=True" not in caplog.text assert "No power curve but has_power_curve=True" not in caplog.text + def test__prepare_and_save_oedb_turbine_curve_data(self, caplog): + """Test `_prepare_and_save_oedb_turbine_curve_data` function.""" + # prepare dummy turbine data + # turbine 0 everything okay, turbine 1 duplicated wind speeds, turbine 2 + # power curve values broken + turbine_data = pd.DataFrame( + data={ + "id": [0, 1, 2], + "turbine_type": ["turbine 0", "turbine 1", "turbine 2"], + "has_power_curve": [True, True, True], + "has_cp_curve": [True, True, True], + "power_curve_wind_speeds": ["[15, 20, 25]", "[15, 15, 25]", "[15, 20, 25]"], + "power_curve_values": ["[15, 20, 25]", "[15, 20, 25]", "[15, 20, [25]"], + "power_coefficient_curve_wind_speeds": ["[15, 20, 25]", "[15, 20, 25]", "[15, 20, 25]"], + "power_coefficient_curve_values": ["[15, 20, 25]", "[15, 20, 25]", "[15, 20, 25]"], + "thrust_coefficient_curve_wind_speeds": [0, 1, 2], + "thrust_coefficient_curve_values": [0, 1, 2], + "nominal_power": [0, 1, 2], + }, + index=[0, 1, 2] + ) + + # run test with low / default threshold - data is not overwritten + t = {} + for fn in os.listdir(self.orig_path): + t[fn] = os.path.getmtime(os.path.join(self.orig_path, fn)) + with caplog.at_level(logging.WARNING): + _process_and_save_oedb_data(turbine_data) + for fn in os.listdir(self.orig_path): + assert t[fn] == os.path.getmtime(os.path.join(self.orig_path, fn)) + assert "The turbine library data contains too many faulty " in caplog.text + + # run test with high threshold + for fn in os.listdir(self.orig_path): + t[fn] = os.path.getmtime(os.path.join(self.orig_path, fn)) + with caplog.at_level(logging.WARNING): + _process_and_save_oedb_data(turbine_data, threshold=0.95) + for fn in os.listdir(self.orig_path): + assert t[fn] < os.path.getmtime(os.path.join(self.orig_path, fn)) + assert "The turbine library data contains faulty power_curves" in caplog.text + assert not turbine_data.at[2, "has_power_curve"] + assert not turbine_data.at[1, "has_power_curve"] + assert turbine_data.at[1, "has_cp_curve"] + assert turbine_data.at[0, "has_power_curve"] + def test_wrong_url_load_turbine_data(self): """Load turbine data from oedb with a wrong schema.""" with pytest.raises( diff --git a/windpowerlib/data.py b/windpowerlib/data.py index 718e4ee3..3562817e 100644 --- a/windpowerlib/data.py +++ b/windpowerlib/data.py @@ -158,7 +158,7 @@ def load_turbine_data_from_oedb(schema="supply", table="wind_turbine_library"): def store_turbine_data_from_oedb( - schema="supply", table="wind_turbine_library" + schema="supply", table="wind_turbine_library", threshold=0.2 ): r""" Loads turbine library from the OpenEnergy database (oedb). @@ -166,6 +166,9 @@ def store_turbine_data_from_oedb( Turbine data is saved to csv files ('oedb_power_curves.csv', 'oedb_power_coefficient_curves.csv' and 'oedb_nominal_power') for offline usage of the windpowerlib. If the files already exist they are overwritten. + In case the turbine library on the oedb contains too many faulty turbines, + the already existing files are not overwritten. The accepted percentage of faulty + turbines can be set through the parameter `threshold`. Parameters ---------- @@ -173,6 +176,16 @@ def store_turbine_data_from_oedb( Database schema of the turbine library. table : str Table name of the turbine library. + threshold : float + In case there are turbines in the turbine library with faulty data (e.g. + duplicate wind speed entries in the power (coefficient) curve data), the + threshold defines the share of accepted faulty turbine ata up to which the + existing turbine data is overwritten by the newly downloaded data. + For example, a threshold of 0.1 means that more than 10% of the + turbines would need to have invalid data in order to discard the downloaded + data. This is to make sure that in the rare case the oedb data is too buggy, + the turbine data that is by default provided with the windpowerlib is not + overwritten by poor data. Returns ------- @@ -182,11 +195,40 @@ def store_turbine_data_from_oedb( """ turbine_data = fetch_turbine_data_from_oedb(schema=schema, table=table) - # standard file name for saving data - filename = os.path.join(os.path.dirname(__file__), "oedb", "{0}.csv") + turbine_data = _process_and_save_oedb_data( + turbine_data, threshold=threshold + ) + check_turbine_data( + filename = os.path.join(os.path.dirname(__file__), "oedb", "{0}.csv") + ) + return turbine_data + + +def _process_and_save_oedb_data(turbine_data, threshold=0.2): + """ + Helper function to extract power (coefficient) curve data from the turbine library. + + Parameters + ----------- + turbine_data : :pandas:`pandas.DataFrame` + Raw turbine data downloaded from the oedb with + :func:`fetch_turbine_data_from_oedb`. + threshold : float + See parameter `threshold` in func:`store_turbine_data_from_oedb` + for more information. + + Returns + -------- + :pandas:`pandas.DataFrame` + Turbine data of different turbines such as 'manufacturer', + 'turbine_type', 'nominal_power'. - # get all power (coefficient) curves and save them to file - for curve_type in ["power_curve", "power_coefficient_curve"]: + """ + curve_types = ["power_curve", "power_coefficient_curve"] + # get all power (coefficient) curves + curve_dict = {} + broken_turbines_dict = {} + for curve_type in curve_types: broken_turbine_data = [] curves_df = pd.DataFrame(columns=["wind_speed"]) for index in turbine_data.index: @@ -222,43 +264,59 @@ def store_turbine_data_from_oedb( curves_df = pd.merge( left=curves_df, right=df, how="outer", on="wind_speed" ) + else: + broken_turbine_data.append( + turbine_data.loc[index, "turbine_type"]) except: broken_turbine_data.append(turbine_data.loc[index, "turbine_type"]) - - # warning in case of broken turbine data - if len(broken_turbine_data) > 0: - issue_link = ("https://github.com/OpenEnergyPlatform/data-preprocessing" - "/issues/28") - # in case only some data is faulty, only give out warning - if len(broken_turbine_data) < 0.2 * len(turbine_data): - logging.warning( - f"The turbine library data contains faulty {curve_type}s. The " - f"{curve_type} data can therefore not be loaded for the following " - f"turbines: {broken_turbine_data}. " - f"Please report this in the following issue, in case it hasn't " - f"already been reported: {issue_link}" - ) - save_turbine_data = True + curve_dict[curve_type] = curves_df + broken_turbines_dict[curve_type] = broken_turbine_data + + # check if there are faulty turbines and if so, raise warning + # if there are too many, don't save downloaded data to disk but keep existing data + if any(len(_) > 0 for _ in broken_turbines_dict.values()): + issue_link = ("https://github.com/OpenEnergyPlatform/data-preprocessing" + "/issues/28") + # in case only some data is faulty, only give out warning + if all(len(_) < threshold * len(turbine_data) + for _ in broken_turbines_dict.values()): + save_turbine_data = True + for curve_type in curve_types: + if len(broken_turbines_dict[curve_type]) > 0: + logging.warning( + f"The turbine library data contains faulty {curve_type}s. The " + f"{curve_type} data can therefore not be loaded for the " + f"following turbines: {broken_turbine_data}. " + f"Please report this in the following issue, in case it hasn't " + f"already been reported: {issue_link}" + ) # set has_power_(coefficient)_curve to False for faulty turbines - for turb in broken_turbine_data: + for turb in broken_turbines_dict[curve_type]: ind = turbine_data[turbine_data.turbine_type == turb].index[0] col = ("has_power_curve" if curve_type == "power_curve" else "has_cp_curve") turbine_data.at[ind, col] = False - # in case most data is faulty, do not store downloaded data - else: - logging.warning( - f"The turbine library data contains too many faulty {curve_type}s," - f"wherefore {curve_type} data is not loaded from the oedb. " - f"Please report this in the following issue, in case it hasn't " - f"already been reported: {issue_link}" - ) - save_turbine_data = False + # in case most data is faulty, do not store downloaded data else: - save_turbine_data = True - - if save_turbine_data: - curves_df = curves_df.set_index("wind_speed").sort_index().transpose() + logging.warning( + f"The turbine library data contains too many faulty turbine datasets " + f"wherefore it is not loaded from the oedb. " + f"In case you want to circumvent this behaviour, you can specify a " + f"higher tolerance through the parameter 'threshold'." + f"Please report this in the following issue, in case it hasn't " + f"already been reported: {issue_link}" + ) + save_turbine_data = False + else: + save_turbine_data = True + + if save_turbine_data: + # standard file name for saving data + filename = os.path.join(os.path.dirname(__file__), "oedb", "{0}.csv") + # save curve data to csv + for curve_type in curve_types: + curves_df = curve_dict[curve_type].set_index( + "wind_speed").sort_index().transpose() # power curve values in W if curve_type == "power_curve": curves_df *= 1000 @@ -266,23 +324,22 @@ def store_turbine_data_from_oedb( curves_df.sort_index(inplace=True) curves_df.to_csv(filename.format("{}s".format(curve_type))) - # get turbine data and save to file (excl. curves) - turbine_data_df = turbine_data.drop( - [ - "power_curve_wind_speeds", - "power_curve_values", - "power_coefficient_curve_wind_speeds", - "power_coefficient_curve_values", - "thrust_coefficient_curve_wind_speeds", - "thrust_coefficient_curve_values", - ], - axis=1, - ).set_index("turbine_type") - # nominal power in W - turbine_data_df["nominal_power"] *= 1000 - turbine_data_df.sort_index(inplace=True) - turbine_data_df.to_csv(filename.format("turbine_data")) - check_turbine_data(filename) + # save turbine data to file (excl. curves) + turbine_data_df = turbine_data.drop( + [ + "power_curve_wind_speeds", + "power_curve_values", + "power_coefficient_curve_wind_speeds", + "power_coefficient_curve_values", + "thrust_coefficient_curve_wind_speeds", + "thrust_coefficient_curve_values", + ], + axis=1, + ).set_index("turbine_type") + # nominal power in W + turbine_data_df["nominal_power"] *= 1000 + turbine_data_df.sort_index(inplace=True) + turbine_data_df.to_csv(filename.format("turbine_data")) return turbine_data