Merge pull request #11 from openclimatefix/Old-Data-Error

OLD-DATA-ERROR

pyproject.toml

@@ -11,6 +11,7 @@ click = "^8.1.7"
 pvsite-datamodel = "^1.0.10"
 pandas = "1.5.3"
 requests = "^2.31.0"
+freezegun = "^1.1.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.4"
@@ -68,7 +69,7 @@ target-version = "py311"
 fix = false
 
 # Group violations by containing file.
-output-format = "github"
+#output-format = "github"
 ignore-init-module-imports = true
 
 [tool.ruff.mccabe]
@@ -86,5 +87,5 @@ convention = "google"
 omit = ["tests/*", "scripts/*"]
 
 [build-system]
-requires = ["poetry-core==1.7.1"]
+requires = ["poetry-core==1.9.0"]
 build-backend = "poetry.core.masonry.api"

ruvnl_consumer_app/app.py

@@ -15,6 +15,7 @@
 
 import click
 import pandas as pd
+import pytz
 import requests
 from pvsite_datamodel import DatabaseConnection, SiteSQL
 from pvsite_datamodel.read import get_sites_by_country
@@ -64,6 +65,7 @@ def fetch_data(data_url: str) -> pd.DataFrame:
     Returns:
             A pandas DataFrame of generation values for wind and PV
     """
+    print("Starting to get data")
     try:
         r = requests.get(data_url, timeout=10)  # 10 seconds
     except requests.exceptions.Timeout as e:
@@ -82,7 +84,16 @@ def fetch_data(data_url: str) -> pd.DataFrame:
 
             start_utc = dt.datetime.fromtimestamp(int(record["SourceTimeSec"]), tz=dt.UTC)
             power_kw = record["Average2"] * 1000  # source is in MW, convert to kW
-
+            if v=="wind":
+                if(start_utc<dt.datetime.now(dt.timezone.utc)-dt.timedelta(hours=1)):
+                        start_ist = start_utc. astimezone(pytz.timezone('Asia/Calcutta'))
+                        start_ist=str(start_ist)
+                        now = dt.datetime.now(pytz.timezone('Asia/Calcutta'))
+                        now = str(now)
+                        timestamp_after_raise = f"Timestamp Now: {now} Timestamp data: {start_ist}"
+                        timestamp_fstring = f"{timestamp_after_raise}"
+                        raise Exception("Start time is at least 1 hour old. " + timestamp_fstring)
+
             data.append({"asset_type": v, "start_utc": start_utc, "power_kw": power_kw})
             log.info(
                 f"Found generation data for asset type: {v}, " f"{power_kw} kW at {start_utc} UTC"
@@ -166,7 +177,7 @@ def app(write_to_db: bool, log_level: str) -> None:
 
     log.info(f'Running data consumer app (version: {__version__})')
 
-    url = os.environ["DB_URL"]
+    url = os.getenv("DB_URL", "sqlite:///test.db")
     data_url = os.getenv("DATA_URL", DEFAULT_DATA_URL)
 
     # 0. Initialise DB connection

scripts/getdata.py

@@ -0,0 +1,6 @@
+""" This is a scratch file, that is useful to run the fetch_data"""
+from ruvnl_consumer_app.app import fetch_data
+
+DEFAULT_DATA_URL = "http://sldc.rajasthan.gov.in/rrvpnl/read-sftp?type=overview"
+data = fetch_data(DEFAULT_DATA_URL)
+print(data)

tests/test_app.py

@@ -7,6 +7,7 @@
 import pandas as pd
 import pytest
 import requests
+from freezegun import freeze_time
 from pvsite_datamodel import GenerationSQL, SiteSQL
 
 from ruvnl_consumer_app.app import (
@@ -58,6 +59,7 @@ class TestFetchData:
     Test suite for fetching data from RUVNL
     """
 
+    @freeze_time("2021-01-31T10:01:00Z")
     def test_fetch_data(self, requests_mock):
         """Test for correctly fetching data"""
 
@@ -82,6 +84,7 @@ def test_fetch_data(self, requests_mock):
         for vals in result[["start_utc", "power_kw"]]:
             assert not pd.isna(vals)
 
+    @freeze_time("2021-01-31T10:01:00Z")
     def test_fetch_data_with_missing_asset(self, requests_mock, caplog):
         """Test for fetching data with missing asset type"""
 
@@ -106,6 +109,17 @@ def test_catch_bad_response_code(self, requests_mock):
         with pytest.raises(requests.exceptions.HTTPError):
             fetch_data(DEFAULT_DATA_URL)
 
+    def test_old_fetch_data(self, requests_mock):
+        """Test for correctly fetching data"""
+
+        requests_mock.get(
+            DEFAULT_DATA_URL,
+            text=load_mock_response("tests/mock/responses/ruvnl-valid-response.json")
+        )
+
+        with pytest.raises(Exception): # noqa
+            fetch_data(DEFAULT_DATA_URL)
+
     def test_catch_bad_response_json(self, requests_mock):
         """Test for catching invalid response JSON"""
 
@@ -181,6 +195,7 @@ def test_save_generation_data(write_to_db, db_session, caplog, associated_genera
         assert "Generation data:" in caplog.text
 
 
+@freeze_time("2021-01-31T10:01:00Z")
 @pytest.mark.parametrize("write_to_db", [True, False])
 def test_app(write_to_db, requests_mock, db_session, caplog):
     """Test for running app from command line"""