chore: Remove lightning_strikes_by_year table, not listed (#215)

* chore: Remove `lightning_strikes_by_year` table, not listed in the datasets marketplace * fix: Remove DAG file
GoogleCloudPlatform · Oct 28, 2021 · a2a8ed9 · a2a8ed9
1 parent c7cdd82
commit a2a8ed9
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 264 deletions.
diff --git a/datasets/noaa/_terraform/gsod_stations_pipeline.tf b/datasets/noaa/_terraform/gsod_stations_pipeline.tf
@@ -15,7 +15,7 @@
  */
 
 
-resource "google_bigquery_table" "gsod_stations" {
+resource "google_bigquery_table" "noaa_gsod_stations" {
   project    = var.project_id
   dataset_id = "noaa"
   table_id   = "gsod_stations"
@@ -30,10 +30,10 @@ resource "google_bigquery_table" "gsod_stations" {
   ]
 }
 
-output "bigquery_table-gsod_stations-table_id" {
-  value = google_bigquery_table.gsod_stations.table_id
+output "bigquery_table-noaa_gsod_stations-table_id" {
+  value = google_bigquery_table.noaa_gsod_stations.table_id
 }
 
-output "bigquery_table-gsod_stations-id" {
-  value = google_bigquery_table.gsod_stations.id
+output "bigquery_table-noaa_gsod_stations-id" {
+  value = google_bigquery_table.noaa_gsod_stations.id
 }
diff --git a/datasets/noaa/_terraform/lightning_strikes_by_year_pipeline.tf b/datasets/noaa/_terraform/lightning_strikes_by_year_pipeline.tf
diff --git a/datasets/noaa/gsod_stations/gsod_stations_dag.py b/datasets/noaa/gsod_stations/gsod_stations_dag.py
@@ -14,26 +14,27 @@
 
 
 from airflow import DAG
-from airflow.contrib.operators import gcs_to_bq, kubernetes_pod_operator
+from airflow.providers.cncf.kubernetes.operators import kubernetes_pod
+from airflow.providers.google.cloud.transfers import gcs_to_bigquery
 
 default_args = {
     "owner": "Google",
     "depends_on_past": False,
-    "start_date": "2021-03-01",
+    "start_date": "2021-12-30",
 }
 
 
 with DAG(
     dag_id="noaa.gsod_stations",
     default_args=default_args,
     max_active_runs=1,
-    schedule_interval="@daily",
+    schedule_interval="@yearly",
     catchup=False,
     default_view="graph",
 ) as dag:
 
     # Run CSV transform within kubernetes pod
-    transform_csv = kubernetes_pod_operator.KubernetesPodOperator(
+    transform_csv = kubernetes_pod.KubernetesPodOperator(
         task_id="transform_csv",
         name="gsod_stations",
         namespace="default",
@@ -70,7 +71,7 @@
     )
 
     # Task to load CSV data to a BigQuery table
-    load_to_bq = gcs_to_bq.GoogleCloudStorageToBigQueryOperator(
+    load_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
         task_id="load_to_bq",
         bucket="{{ var.value.composer_bucket }}",
         source_objects=["data/noaa/gsod_stations/data_output.csv"],

diff --git a/datasets/noaa/lightning_strikes_by_year/lightning_strikes_by_year_dag.py b/datasets/noaa/lightning_strikes_by_year/lightning_strikes_by_year_dag.py
diff --git a/datasets/noaa/lightning_strikes_by_year/pipeline.yaml b/datasets/noaa/lightning_strikes_by_year/pipeline.yaml