851 fix test prevent dtype int testcase (#856)

* avoid local data usage

* fixed and added testcases

tests/conftest.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu May 16 15:51:44 2024
+
+@author: veenstra
+"""
+import pytest
+import dfm_tools as dfmt
+import os
+
+
+@pytest.mark.requiressecrets
+@pytest.fixture
+def file_nc_era5_pattern(tmp_path):
+    date_min = '2010-01-31'
+    date_max = '2010-02-01'
+    longitude_min, longitude_max, latitude_min, latitude_max =    2,   3,  51, 52 #test domain
+    variables_era5 = ['msl']#'v10n'] # check variables_dict in dfmt.download_ERA5() for valid names
+    for varkey in variables_era5:
+        dfmt.download_ERA5(varkey, 
+                           longitude_min=longitude_min, longitude_max=longitude_max, latitude_min=latitude_min, latitude_max=latitude_max,
+                           date_min=date_min, date_max=date_max,
+                           dir_output=tmp_path, overwrite=True)
+
+    # assert downloaded files
+    file_nc_era5_pattern = os.path.join(tmp_path, "*.nc")
+    return file_nc_era5_pattern

tests/test_download.py

@@ -11,6 +11,7 @@
 from dfm_tools.download import cds_credentials, copernicusmarine_credentials
 import dfm_tools as dfmt
 import xarray as xr
+import glob
 
 
 @pytest.mark.requiressecrets
@@ -32,23 +33,15 @@ def test_copernicusmarine_credentials():
 
 @pytest.mark.requiressecrets
 @pytest.mark.unittest
-def test_download_era5(tmp_path):
-    date_min = '2010-01-01'
-    date_max = '2010-01-02'
-    longitude_min, longitude_max, latitude_min, latitude_max =    2,   3,  51, 52 #test domain
-    variables_era5 = ['msl']#'v10n'] # check variables_dict in dfmt.download_ERA5() for valid names
-    for varkey in variables_era5:
-        dfmt.download_ERA5(varkey, 
-                           longitude_min=longitude_min, longitude_max=longitude_max, latitude_min=latitude_min, latitude_max=latitude_max,
-                           date_min=date_min, date_max=date_max,
-                           dir_output=tmp_path, overwrite=True)
+def test_download_era5(file_nc_era5_pattern):
+    # file_nc_era5_pattern comes from conftest.py
+    file_list = glob.glob(file_nc_era5_pattern)
+    assert len(file_list) == 2
 
-    # assert downloaded files
-    file_nc_pat = os.path.join(tmp_path, "*.nc")
-    ds = xr.open_mfdataset(file_nc_pat)
-    assert ds.sizes["time"] == 744
+    ds = xr.open_mfdataset(file_nc_era5_pattern)
+    assert ds.sizes["time"] == 1416
     assert ds.time.to_pandas().iloc[0] == pd.Timestamp('2010-01-01')
-    assert ds.time.to_pandas().iloc[-1] == pd.Timestamp('2010-01-31 23:00')
+    assert ds.time.to_pandas().iloc[-1] == pd.Timestamp('2010-02-28 23:00')
 
 
 @pytest.mark.requiressecrets

tests/test_xarray_helpers.py

@@ -11,36 +11,49 @@
 import xarray as xr
 import numpy as np
 from dfm_tools.xarray_helpers import prevent_dtype_int
+import dfm_tools as dfmt
+import pandas as pd
 
 #TODO: many xarray_helpers tests are still in test_dfm_tools.py
 
 
-@pytest.mark.requireslocaldata
+@pytest.mark.requiressecrets
 @pytest.mark.unittest
-def test_prevent_dtype_int(tmp_path):
-    #open data
-    dir_data = r'p:\11207892-pez-metoceanmc\3D-DCSM-FM\workflow_manual\01_scripts\04_meteo\era5_temp'
-    file_nc = os.path.join(dir_data,'era5_mslp_*.nc')
+def test_prevent_dtype_int(tmp_path, file_nc_era5_pattern):
+    # file_nc_era5_pattern comes from file_nc_era5_pattern() in conftest.py
+    varn = "msl"
+    file_nc = os.path.join(tmp_path,f'era5_{varn}_*.nc')
 
     #optional encoding
-    for zlib, size_expected in zip([False, True], [100e6, 50e6]):
+    for zlib, size_expected in zip([False, True], [480000, 250000]):
         data_xr = xr.open_mfdataset(file_nc)
         prevent_dtype_int(data_xr, zlib=zlib)
 
         #write to netcdf file
-        file_out = os.path.join(tmp_path, 'era5_mslp_prevent_dtype_int.nc')
+        file_out = os.path.join(tmp_path, f'era5_prevent_dtype_int_{varn}_zlib_{zlib}.nc')
         data_xr.to_netcdf(file_out)
         data_xr_check = xr.open_dataset(file_out)
-        print(data_xr_check.msl.encoding)
+        print(data_xr_check[varn].encoding)
 
-        absdiff = (data_xr_check - data_xr).apply(np.fabs)
-        absdiff_max = absdiff.msl.max(dim=['longitude','latitude'])
+        assert np.allclose(data_xr[varn], data_xr_check[varn])
 
-        assert np.allclose(absdiff_max, 0)
-
         del data_xr
         del data_xr_check
 
         file_size = os.path.getsize(file_out)
         print(file_size)
         assert file_size < size_expected
+
+
+@pytest.mark.requiressecrets
+@pytest.mark.unittest
+def test_merge_meteofiles(file_nc_era5_pattern):
+    # file_nc_era5_pattern comes from file_nc_era5_pattern() in conftest.py
+    ds = dfmt.merge_meteofiles(file_nc=file_nc_era5_pattern, 
+                               preprocess=dfmt.preprocess_ERA5, 
+                               time_slice=slice("2010-01-30","2010-02-01")
+                               )
+    assert ds.sizes["time"] == 72
+    assert ds.time.to_pandas().iloc[0] == pd.Timestamp('2010-01-30')
+    assert ds.time.to_pandas().iloc[-1] == pd.Timestamp('2010-02-01 23:00')
+    assert "msl" in ds.data_vars