fix(integration-test): separate dswx-s1 tests

Refs #705
nasa · Mar 1, 2024 · f8d779f · f8d779f
1 parent 742127b
commit f8d779f
Show file tree

Hide file tree

Showing 4 changed files with 100 additions and 62 deletions.
diff --git a/data_subscriber/rtc/evaluator.py b/data_subscriber/rtc/evaluator.py
@@ -1,4 +1,5 @@
 import argparse
+import json
 import logging
 import re
 import sys
@@ -202,5 +203,10 @@ def join_product_file_docs(result_set_id_to_product_sets_map, product_id_to_prod
     parser = argparse.ArgumentParser()
     parser.add_argument("--coverage-target", type=int, default=100)
     parser.add_argument("--rtc-product-ids", nargs="*")
+    parser.add_argument("--main", action="store_true", default=False)
     args = parser.parse_args(sys.argv[1:])
-    evaluate_rtc_products(**vars(args))
+    if args.main:
+        evaluator_results = main(coverage_target=args.coverage_target)
+        print(json.dumps(evaluator_results))
+    else:
+        evaluate_rtc_products(**vars(args))
diff --git a/integration/test_dswx_s1_edge_cases.py b/integration/test_dswx_s1_edge_cases.py
@@ -0,0 +1,90 @@
+import json
+import logging
+import subprocess
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import dateutil
+import pytest
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.asyncio
+async def test_subscriber_rtc_trigger_logic():
+    mgrs_set_ids_dt = [
+        ("MS_20_29", '20231101T013115Z'),
+        ("MS_33_26", '20231101T225251Z'),
+        ("MS_135_25", '20231108T224433Z'),
+        ("MS_4_8", '20231111T230027Z'),
+        ("MS_4_15", '20231111T230348Z'),
+        ("MS_33_13", '20231101T224618Z'),
+        ("MS_74_46", '20231023T183051Z'),
+        ("MS_1_59", '20231111T183217Z'),
+        ("MS_26_48", '20231101T113548Z')
+    ]
+    for mgrs_set_id, acq_dts in mgrs_set_ids_dt:
+        dt = dateutil.parser.parse(acq_dts)  #.strftime("%Y%m%dT%H%M%SZ")
+        if mgrs_set_id == "MS_74_46":
+            start_dt: datetime = dt - timedelta(minutes=2)
+            end_dt: datetime = dt + timedelta(minutes=2)
+        else:
+            start_dt: datetime = dt - timedelta(minutes=1)
+            end_dt: datetime = dt + timedelta(minutes=1)
+
+        start_dts = start_dt.strftime("%Y%m%dT%H%M%SZ")
+        end_dts = end_dt.strftime("%Y%m%dT%H%M%SZ")
+        print(start_dts, end_dts)
+
+        logger.info("Running DAAC data subscriber")
+
+        result = subprocess.run([
+                "python data_subscriber/daac_data_subscriber.py query "
+                "--endpoint=OPS "
+                "--collection-shortname=OPERA_L2_RTC-S1_V1 "
+                f'--start-date={start_dt.isoformat(timespec="seconds").replace("+00:00", "Z")} '
+                f'--end-date={end_dt.isoformat(timespec="seconds").replace("+00:00", "Z")} '
+                "--transfer-protocol=https "
+                "--chunk-size=1 "
+                "--use-temporal "
+                ""
+            ],
+            cwd=Path.cwd(),
+            shell=True,
+            text=True,
+            check=True,
+            capture_output=True
+        )
+        print(result.stdout)
+        print(result.stderr)
+        logger.info("Ran DAAC data subscriber")
+
+    logger.info("Run evaluator")
+    result = subprocess.run([
+            "python data_subscriber/rtc/evaluator.py "
+            "--coverage-target=0 "
+            "--main "
+            ""
+        ],
+        cwd=Path.cwd(),
+        shell=True,
+        text=True,
+        check=True,
+        capture_output=True
+    )
+    logger.info("Ran evaluator")
+    print(result.stdout)
+    print(result.stderr)
+
+    result = json.loads(result.stdout)
+    print(result)
+
+    assert result["mgrs_sets"]["MS_20_29"][0]["coverage_actual"] == 2
+    assert result["mgrs_sets"]["MS_33_26"][0]["coverage_actual"] == 80
+    assert result["mgrs_sets"]["MS_135_25"][0]["coverage_actual"] == 77
+    assert not result["mgrs_sets"].get("MS_4_8")
+    assert not result["mgrs_sets"].get("MS_4_15")
+    assert not result["mgrs_sets"].get("MS_33_13")
+    assert result["mgrs_sets"]["MS_74_46"][0]["coverage_actual"] == 29
+    assert not result["mgrs_sets"].get("MS_1_59")
+    assert result["mgrs_sets"]["MS_26_48"][0]["coverage_actual"] == 60
diff --git a/integration/test_integration.py b/integration/test_integration.py
@@ -1,19 +1,14 @@
 import logging
 import re
 import time
-from datetime import timedelta, datetime
-
-import dateutil
-import pytest
 
 import conftest
-from data_subscriber import daac_data_subscriber
 from int_test_util import \
     mock_cnm_r_success_sns, \
     mock_cnm_r_success_sqs, \
     wait_for_cnm_s_success, \
     wait_for_cnm_r_success, \
-    wait_for_l3, es_index_delete_by_prefix
+    wait_for_l3
 from subscriber_util import \
     wait_for_query_job, \
     wait_for_download_job, \
@@ -508,60 +503,6 @@ def test_subscriber_rtc():
     assert_cnm_r_success(response_dswx_s1_12)
 
 
-@pytest.mark.asyncio
-async def test_subscriber_rtc_trigger_logic():
-    mgrs_set_ids_dt = [
-        ("MS_20_29", '20231101T013115Z'),
-        ("MS_33_26", '20231101T225251Z'),
-        ("MS_135_25", '20231108T224433Z'),
-        ("MS_4_8", '20231111T230027Z'),
-        ("MS_4_15", '20231111T230348Z'),
-        ("MS_33_13", '20231101T224618Z'),
-        ("MS_74_46", '20231023T183051Z'),
-        ("MS_1_59", '20231111T183217Z'),
-        ("MS_26_48", '20231101T113548Z')
-    ]
-    for mgrs_set_id, acq_dts in mgrs_set_ids_dt:
-        dt = dateutil.parser.parse(acq_dts)  #.strftime("%Y%m%dT%H%M%SZ")
-        if mgrs_set_id == "MS_74_46":
-            start_dt: datetime = dt - timedelta(minutes=2)
-            end_dt: datetime = dt + timedelta(minutes=2)
-        else:
-            start_dt: datetime = dt - timedelta(minutes=1)
-            end_dt: datetime = dt + timedelta(minutes=1)
-
-        start_dts = start_dt.strftime("%Y%m%dT%H%M%SZ")
-        end_dts = end_dt.strftime("%Y%m%dT%H%M%SZ")
-        print(start_dts, end_dts)
-
-        args = "dummy.py query " \
-               "--endpoint=OPS " \
-               "--collection-shortname=OPERA_L2_RTC-S1_V1 " \
-               f'--start-date={start_dt.isoformat(timespec="seconds").replace("+00:00", "Z")} ' \
-               f'--end-date={end_dt.isoformat(timespec="seconds").replace("+00:00", "Z")} ' \
-               "--transfer-protocol=https " \
-               "--chunk-size=1 " \
-               "--use-temporal " \
-               "".split()
-
-        # ACT
-        await daac_data_subscriber.run(args)  # call to populate GRQ
-
-    from data_subscriber.rtc import evaluator
-    result = evaluator.main(coverage_target=0)
-    assert result["mgrs_sets"]["MS_20_29"][0]["coverage_actual"] == 2
-    assert result["mgrs_sets"]["MS_33_26"][0]["coverage_actual"] == 80
-    assert result["mgrs_sets"]["MS_135_25"][0]["coverage_actual"] == 77
-    assert not result["mgrs_sets"].get("MS_4_8")
-    assert not result["mgrs_sets"].get("MS_4_15")
-    assert not result["mgrs_sets"].get("MS_33_13")
-    assert result["mgrs_sets"]["MS_74_46"][0]["coverage_actual"] == 29
-    assert not result["mgrs_sets"].get("MS_1_59")
-    assert result["mgrs_sets"]["MS_26_48"][0]["coverage_actual"] == 60
-
-    es_index_delete_by_prefix("rtc_catalog")
-
-
 def assert_cnm_s_success(response):
     assert response.hits.hits[0]["_source"]["daac_CNM_S_status"] == "SUCCESS"
     assert response.hits.hits[0]["_source"]["daac_CNM_S_timestamp"] is not None

diff --git a/pip.conf b/pip.conf
@@ -8,4 +8,5 @@
 
 [global]
 # passthrough proxy and cache for PyPI.
-index-url = https://artifactory.jpl.nasa.gov/artifactory/api/pypi/pypi-release-virtual/simple
+index-url = https://artifactory.jpl.nasa.gov/artifactory/api/pypi/pypi-release-virtual/simple
+extra-index-url = https://pypi.org/simple