Fix: Renamed dataset from sunroof to sunroof_solar (#226)

* renamed sunroof dataset to sunroof_solar * fix: make conventional commits pass
GoogleCloudPlatform · Nov 12, 2021 · 0780df8 · 0780df8
1 parent a3f4b34
commit 0780df8
Show file tree

Hide file tree

Showing 14 changed files with 64 additions and 63 deletions.
diff --git a/datasets/sunroof/_terraform/sunroof_dataset.tf b/datasets/sunroof/_terraform/sunroof_dataset.tf
diff --git a/.../_images/run_csv_transform_kub/Dockerfile → .../_images/run_csv_transform_kub/Dockerfile b/.../_images/run_csv_transform_kub/Dockerfile → .../_images/run_csv_transform_kub/Dockerfile
diff --git a/...es/run_csv_transform_kub/csv_transform.py → ...es/run_csv_transform_kub/csv_transform.py b/...es/run_csv_transform_kub/csv_transform.py → ...es/run_csv_transform_kub/csv_transform.py
diff --git a/...es/run_csv_transform_kub/requirements.txt → ...es/run_csv_transform_kub/requirements.txt b/...es/run_csv_transform_kub/requirements.txt → ...es/run_csv_transform_kub/requirements.txt
diff --git a/datasets/sunroof/_terraform/provider.tf → ...sets/sunroof_solar/_terraform/provider.tf b/datasets/sunroof/_terraform/provider.tf → ...sets/sunroof_solar/_terraform/provider.tf
diff --git a/...lar_potential_by_census_tract_pipeline.tf → ...olar_potential_by_censustract_pipeline.tf b/...lar_potential_by_census_tract_pipeline.tf → ...olar_potential_by_censustract_pipeline.tf
@@ -15,25 +15,25 @@
  */
 
 
-resource "google_bigquery_table" "solar_potential_by_census_tract" {
+resource "google_bigquery_table" "sunroof_solar_solar_potential_by_censustract" {
   project    = var.project_id
-  dataset_id = "sunroof"
-  table_id   = "solar_potential_by_census_tract"
+  dataset_id = "sunroof_solar"
+  table_id   = "solar_potential_by_censustract"
 
   description = "Sunroof Solar Potential By Census Tract"
 
 
 
 
   depends_on = [
-    google_bigquery_dataset.sunroof
+    google_bigquery_dataset.sunroof_solar
   ]
 }
 
-output "bigquery_table-solar_potential_by_census_tract-table_id" {
-  value = google_bigquery_table.solar_potential_by_census_tract.table_id
+output "bigquery_table-sunroof_solar_solar_potential_by_censustract-table_id" {
+  value = google_bigquery_table.sunroof_solar_solar_potential_by_censustract.table_id
 }
 
-output "bigquery_table-solar_potential_by_census_tract-id" {
-  value = google_bigquery_table.solar_potential_by_census_tract.id
+output "bigquery_table-sunroof_solar_solar_potential_by_censustract-id" {
+  value = google_bigquery_table.sunroof_solar_solar_potential_by_censustract.id
 }
diff --git a/...olar_potential_by_postal_code_pipeline.tf → ...olar_potential_by_postal_code_pipeline.tf b/...olar_potential_by_postal_code_pipeline.tf → ...olar_potential_by_postal_code_pipeline.tf
@@ -15,9 +15,9 @@
  */
 
 
-resource "google_bigquery_table" "solar_potential_by_postal_code" {
+resource "google_bigquery_table" "sunroof_solar_solar_potential_by_postal_code" {
   project    = var.project_id
-  dataset_id = "sunroof"
+  dataset_id = "sunroof_solar"
   table_id   = "solar_potential_by_postal_code"
 
   description = "Sunroof Solar Potential By Postal Code"
@@ -26,14 +26,14 @@ resource "google_bigquery_table" "solar_potential_by_postal_code" {
 
 
   depends_on = [
-    google_bigquery_dataset.sunroof
+    google_bigquery_dataset.sunroof_solar
   ]
 }
 
-output "bigquery_table-solar_potential_by_postal_code-table_id" {
-  value = google_bigquery_table.solar_potential_by_postal_code.table_id
+output "bigquery_table-sunroof_solar_solar_potential_by_postal_code-table_id" {
+  value = google_bigquery_table.sunroof_solar_solar_potential_by_postal_code.table_id
 }
 
-output "bigquery_table-solar_potential_by_postal_code-id" {
-  value = google_bigquery_table.solar_potential_by_postal_code.id
+output "bigquery_table-sunroof_solar_solar_potential_by_postal_code-id" {
+  value = google_bigquery_table.sunroof_solar_solar_potential_by_postal_code.id
 }
diff --git a/datasets/sunroof_solar/_terraform/sunroof_solar_dataset.tf b/datasets/sunroof_solar/_terraform/sunroof_solar_dataset.tf
@@ -0,0 +1,26 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+resource "google_bigquery_dataset" "sunroof_solar" {
+  dataset_id  = "sunroof_solar"
+  project     = var.project_id
+  description = "Project Sunroof computes how much sunlight hits your roof in a year. Solar viability is determined using a methodology found here: https://www.google.com/get/sunroof/data-explorer/data-explorer-methodology.pdf The use of this data is subject to Google\u0027s Terms of Service. Feel free to include this data from Project Sunroof in other analyses, materials, reports, and communications with the following attribution: Source: Google Project Sunroof data explorer (August 2017)."
+}
+
+output "bigquery_dataset-sunroof_solar-dataset_id" {
+  value = google_bigquery_dataset.sunroof_solar.dataset_id
+}
diff --git a/datasets/sunroof/_terraform/variables.tf → ...ets/sunroof_solar/_terraform/variables.tf b/datasets/sunroof/_terraform/variables.tf → ...ets/sunroof_solar/_terraform/variables.tf
diff --git a/datasets/sunroof/dataset.yaml → datasets/sunroof_solar/dataset.yaml b/datasets/sunroof/dataset.yaml → datasets/sunroof_solar/dataset.yaml
@@ -13,14 +13,15 @@
 # limitations under the License.
 
 dataset:
-  name: sunroof
-  friendly_name: sunroof
-  description: sunroof
+  name: sunroof_solar
+  friendly_name: sunroof_solar
+  description: sunroof_solar
   dataset_sources: ~
   terms_of_use: ~
 
 
 resources:
   - type: bigquery_dataset
-    dataset_id: sunroof
-    description: sunroof
+    dataset_id: sunroof_solar
+    description: |-
+      Project Sunroof computes how much sunlight hits your roof in a year. Solar viability is determined using a methodology found here: https://www.google.com/get/sunroof/data-explorer/data-explorer-methodology.pdf The use of this data is subject to Google's Terms of Service. Feel free to include this data from Project Sunroof in other analyses, materials, reports, and communications with the following attribution: Source: Google Project Sunroof data explorer (August 2017).
diff --git a/...r_potential_by_census_tract/pipeline.yaml → ...ar_potential_by_censustract/pipeline.yaml b/...r_potential_by_census_tract/pipeline.yaml → ...ar_potential_by_censustract/pipeline.yaml
@@ -16,13 +16,13 @@
 resources:
 
   - type: bigquery_table
-    table_id: "solar_potential_by_census_tract"
+    table_id: "solar_potential_by_censustract"
     description: "Sunroof Solar Potential By Census Tract"
 
 dag:
   airflow_version: 2
   initialize:
-    dag_id: solar_potential_by_census_tract
+    dag_id: solar_potential_by_censustract
     default_args:
       owner: "Google"
       depends_on_past: False
@@ -40,7 +40,7 @@ dag:
       args:
 
         task_id: "transform_csv"
-        name: "solar_potential_by_census_tract"
+        name: "solar_potential_by_censustract"
         namespace: "default"
         affinity:
           nodeAffinity:
@@ -52,27 +52,27 @@ dag:
                       values:
                         - "pool-e2-standard-4"
         image_pull_policy: "Always"
-        image: "{{ var.json.sunroof.container_registry.run_csv_transform_kub }}"
+        image: "{{ var.json.sunroof_solar.container_registry.run_csv_transform_kub }}"
         env_vars:
           SOURCE_URL: "gs://project-sunroof/csv/latest/project-sunroof-census_tract.csv"
           SOURCE_FILE: "files/data.csv"
           TARGET_FILE: "files/data_output.csv"
           CHUNKSIZE: "750000"
           TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}"
-          TARGET_GCS_PATH: "data/sunroof/solar_potential_by_census_tract/data_output.csv"
+          TARGET_GCS_PATH: "data/sunroof/solar_potential_by_censustract/data_output.csv"
         resources:
           limit_memory: "8G"
           limit_cpu: "3"
 
     - operator: "GoogleCloudStorageToBigQueryOperator"
-      description: "Task to load CSV data to a BigQuery table"
+      description: "Load CSV data to a BigQuery table"
 
       args:
         task_id: "load_to_bq"
         bucket: "{{ var.value.composer_bucket }}"
-        source_objects: ["data/sunroof/solar_potential_by_census_tract/data_output.csv"]
+        source_objects: ["data/sunroof/solar_potential_by_censustract/data_output.csv"]
         source_format: "CSV"
-        destination_project_dataset_table: "sunroof.solar_potential_by_census_tract"
+        destination_project_dataset_table: "sunroof.solar_potential_by_censustract"
         skip_leading_rows: 1
         allow_quoted_newlines: True
         write_disposition: "WRITE_TRUNCATE"

diff --git a/...ct/solar_potential_by_census_tract_dag.py → ...act/solar_potential_by_censustract_dag.py b/...ct/solar_potential_by_census_tract_dag.py → ...act/solar_potential_by_censustract_dag.py
@@ -25,7 +25,7 @@
 
 
 with DAG(
-    dag_id="sunroof.solar_potential_by_census_tract",
+    dag_id="sunroof_solar.solar_potential_by_censustract",
     default_args=default_args,
     max_active_runs=1,
     schedule_interval="@daily",
@@ -36,7 +36,7 @@
     # Run CSV transform within kubernetes pod
     transform_csv = kubernetes_pod.KubernetesPodOperator(
         task_id="transform_csv",
-        name="solar_potential_by_census_tract",
+        name="solar_potential_by_censustract",
         namespace="default",
         affinity={
             "nodeAffinity": {
@@ -56,14 +56,14 @@
             }
         },
         image_pull_policy="Always",
-        image="{{ var.json.sunroof.container_registry.run_csv_transform_kub }}",
+        image="{{ var.json.sunroof_solar.container_registry.run_csv_transform_kub }}",
         env_vars={
             "SOURCE_URL": "gs://project-sunroof/csv/latest/project-sunroof-census_tract.csv",
             "SOURCE_FILE": "files/data.csv",
             "TARGET_FILE": "files/data_output.csv",
             "CHUNKSIZE": "750000",
             "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}",
-            "TARGET_GCS_PATH": "data/sunroof/solar_potential_by_census_tract/data_output.csv",
+            "TARGET_GCS_PATH": "data/sunroof/solar_potential_by_censustract/data_output.csv",
         },
         resources={"limit_memory": "8G", "limit_cpu": "3"},
     )
@@ -72,9 +72,9 @@
     load_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
         task_id="load_to_bq",
         bucket="{{ var.value.composer_bucket }}",
-        source_objects=["data/sunroof/solar_potential_by_census_tract/data_output.csv"],
+        source_objects=["data/sunroof/solar_potential_by_censustract/data_output.csv"],
         source_format="CSV",
-        destination_project_dataset_table="sunroof.solar_potential_by_census_tract",
+        destination_project_dataset_table="sunroof.solar_potential_by_censustract",
         skip_leading_rows=1,
         allow_quoted_newlines=True,
         write_disposition="WRITE_TRUNCATE",

diff --git a/...ar_potential_by_postal_code/pipeline.yaml → ...ar_potential_by_postal_code/pipeline.yaml b/...ar_potential_by_postal_code/pipeline.yaml → ...ar_potential_by_postal_code/pipeline.yaml
@@ -52,7 +52,7 @@ dag:
                       values:
                         - "pool-e2-standard-4"
         image_pull_policy: "Always"
-        image: "{{ var.json.sunroof.container_registry.run_csv_transform_kub }}"
+        image: "{{ var.json.sunroof_solar.container_registry.run_csv_transform_kub }}"
         env_vars:
           SOURCE_URL: "gs://project-sunroof/csv/latest/project-sunroof-postal_code.csv"
           SOURCE_FILE: "files/data.csv"

diff --git a/...ode/solar_potential_by_postal_code_dag.py → ...ode/solar_potential_by_postal_code_dag.py b/...ode/solar_potential_by_postal_code_dag.py → ...ode/solar_potential_by_postal_code_dag.py
@@ -25,7 +25,7 @@
 
 
 with DAG(
-    dag_id="sunroof.solar_potential_by_postal_code",
+    dag_id="sunroof_solar.solar_potential_by_postal_code",
     default_args=default_args,
     max_active_runs=1,
     schedule_interval="@daily",
@@ -56,7 +56,7 @@
             }
         },
         image_pull_policy="Always",
-        image="{{ var.json.sunroof.container_registry.run_csv_transform_kub }}",
+        image="{{ var.json.sunroof_solar.container_registry.run_csv_transform_kub }}",
         env_vars={
             "SOURCE_URL": "gs://project-sunroof/csv/latest/project-sunroof-postal_code.csv",
             "SOURCE_FILE": "files/data.csv",