From 5e27a9d19b2105f727686065c9ca9651ae17f3c7 Mon Sep 17 00:00:00 2001 From: Savannah Goodman Date: Fri, 20 Aug 2021 14:20:44 -0700 Subject: [PATCH 1/5] feat: adds cfe dataset --- .../google_cfe/datacenter_cfe/pipeline.yaml | 143 ++++++++++++++++++ datasets/google_cfe/dataset.yaml | 83 ++++++++++ 2 files changed, 226 insertions(+) create mode 100644 datasets/google_cfe/datacenter_cfe/pipeline.yaml create mode 100644 datasets/google_cfe/dataset.yaml diff --git a/datasets/google_cfe/datacenter_cfe/pipeline.yaml b/datasets/google_cfe/datacenter_cfe/pipeline.yaml new file mode 100644 index 000000000..7474e19d4 --- /dev/null +++ b/datasets/google_cfe/datacenter_cfe/pipeline.yaml @@ -0,0 +1,143 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +--- +resources: + # A list of GCP resources that are unique and specific to your pipeline. + # + # The currently supported resources are shown below. Use only the resources + # needed by your pipeline, and delete the rest of the examples. + # + # We will keep adding to the list below to support more Google Cloud resources + # over time. If a resource you need isn't supported, please file an issue on + # the repository. + + - type: bigquery_table + # A Google BigQuery table to store your data. Requires a `bigquery_dataset` + # to be specified in the config (i.e. `dataset.yaml) for the dataset that + # this pipeline belongs in. + # + # Required Properties: + # table_id + table_id: datacenter_cfe + + # Optional Properties: + # Description of the table + description: ~ + +dag: + # [Required] Specify the Airflow version of the operators used by the DAG. + airflow_version: 2 + + # The DAG acronym stands for directed acyclic graph. This block represents + # your data pipeline along with every property and configuration it needs to + # onboard your data. + initialize: + dag_id: PIPELINE_FOLDER_NAME + default_args: + owner: "Google" + + # When set to True, keeps a task from getting triggered if the previous schedule for the task hasn’t succeeded + depends_on_past: False + start_date: '2021-03-01' + max_active_runs: 1 + schedule_interval: "@once" + catchup: False + default_view: graph + + tasks: + # This is where you specify the tasks (a.k.a. processes) that your data + # pipeline will run to onboard the data. + # + # As the examples below will show, every task must be represented by an + # Airflow operator. The list of suported operators are listed in + # + # scripts/dag_imports.json + # + # If an operator you need isn't supported, please file an issue on the + # repository. + # + # Use the YAML list syntax in this block to specify every task for your + # pipeline. + + + - operator: "GoogleCloudStorageToBigQueryOperator" + # Initializes GCS to BQ task for the DAG. This operator is used to load a + # JSON, CSV, Avro, ORC, or Parquet data from GCS into a BigQuery table. + + # Task description + description: "Task to load CSV data to a BigQuery table" + + # Arguments supported by this operator: + # http://airflow.apache.org/docs/apache-airflow/stable/howto/operator/gcp/gcs.html#googlecloudstoragetobigqueryoperator + args: + task_id: "cfe_gcs_to_bq" + + # The GCS bucket where the CSV file is located in. + bucket: "{{ var.json.shared.composer_bucket }}" + + # The GCS object path for the CSV file + source_objects: ["data/DATASET_FOLDER_NAME/PIPELINE_FOLDER_NAME/run_date={{ ds }}/data.csv"] + source_format: "CSV" + destination_project_dataset_table: "google_cfe.datacenter_cfe" + + # Use this if your CSV file contains a header row + skip_leading_rows: 1 + + # How to write data to the table: overwrite, append, or write if empty + # See https://cloud.google.com/bigquery/docs/reference/auditlogs/rest/Shared.Types/WriteDisposition + write_disposition: "WRITE_TRUNCATE" + + # The BigQuery table schema based on the CSV file. For more info, see + # https://cloud.google.com/bigquery/docs/schemas. + # Always use snake_case and lowercase for column names, and be explicit, + # i.e. specify modes for all columns. + schema_fields: + - name: "year" + type: "INTEGER" + mode: "REQUIRED" + description: "The year for which the Google CFE metric has been aggregated. We will continue to add new data for each year as we make progress. Note that the Google CFE metric will only be included for the years that the Cloud Region or Data Center was operational. For example, the Data Center in Denmark came online in 2020, so the Google CFE data for that region starts in 2020 (and there is no data for Denmark for 2019)." + - name: "cfe_region" + type: "STRING" + mode: "REQUIRED" + description: "The regional boundary of the electric grid we consider when calculating the Google CFE score. For most of the world, the CFE region is defined as the country. However, for countries that have distinct grids, such as the US, there are multiple CFE regions, which are defined by the balancing authority." + - name: "zone_id" + type: "STRING" + mode: "REQUIRED" + description: "This is the ID associated with the CFE Region based on Tomorrow's ElectricityMap API definition. (http://static.electricitymap.org/api/docs/index.html#zones)" + - name: "cloud_region" + type: "STRING" + mode: "REQUIRED" + description: "The Google Cloud Region that is mapped to the CFE region. For Google Data Centers that are not Cloud Regions, the region will be labeled 'non-cloud-data-center'." + - name: "location" + type: "STRING" + mode: "REQUIRED" + description: "This is the "friendly name" of the Cloud Region. For Google Data Centers that are not Cloud regions, the location will be the country (non-US) or the state (US) that the Data Center is located in." + - name: "google_cfe" + type: "FLOAT" + mode: "NULLABLE" + description: "This metric is calculated for every hour in every region and tells us what percentage of the energy we consumed during an hour that is carbon-free. We take into account the carbon-free energy that’s already supplied by the grid, in addition to the investments we have made in renewable energy in that location to reach our 24/7 carbon-free objective (https://www.gstatic.com/gumdrop/sustainability/247-carbon-free-energy.pdf). We then aggregate the available average hourly CFE percentage for each region for the year. We do not currently have the hourly energy information available for calculating the metrics for all regions. We anticipate rolling out the calculated metrics using hourly data to regions as the data becomes available." + + # Initializes a GCS-to-GCS task for the DAG. This operator is used to copy or move + # GCS objects from one location to another. + + graph_paths: + # This is where you specify the relationships (i.e. directed paths/edges) + # among the tasks specified above. Use the bitshift operator to define the + # relationships and the `task_id` value above to represent tasks. + # + # For more info, see + # https://airflow.apache.org/docs/apache-airflow/stable/tutorial.html#setting-up-dependencies + - "cfe_gcs_to_bq" \ No newline at end of file diff --git a/datasets/google_cfe/dataset.yaml b/datasets/google_cfe/dataset.yaml new file mode 100644 index 000000000..ec778d8d4 --- /dev/null +++ b/datasets/google_cfe/dataset.yaml @@ -0,0 +1,83 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dataset: + # The `dataset` block includes properties for your dataset that will be shown + # to users of your data on the Google Cloud website. + + # Must be exactly the same name as the folder name your dataset.yaml is in. + name: google_cfe + + # A friendly, human-readable name of the dataset + friendly_name: ~ + + # A short, descriptive summary of the dataset. + description: ~ + + # A list of sources the dataset is derived from, using the YAML list syntax. + dataset_sources: ~ + + # A list of terms and conditions that users of the dataset should agree on, + # using the YAML list syntax. + terms_of_use: ~ + + +resources: + # A list of Google Cloud resources needed by your dataset. In principle, all + # pipelines under a dataset should be able to share these resources. + # + # The currently supported resources are shown below. Use only the resources + # you need, and delete the rest as needed by your pipeline. + # + # We will keep adding to the list below to support more Google Cloud resources + # over time. If a resource you need isn't supported, please file an issue on + # the repository. + + - type: bigquery_dataset + # Google BigQuery dataset to namespace all tables managed by this folder + # + # Required Properties: + # dataset_id + # + # Optional Properties: + # friendly_name (A user-friendly name of the dataset) + # description (A user-friendly description of the dataset) + # location (The geographic location where the dataset should reside) + dataset_id: google_cfe + description: ~ + + - type: storage_bucket + # Google Cloud Storage Bucket that your pipelines need. Say, you need an + # intermediate bucket to store data in-flight. Or you need a bucket to + # archive or backup data generated by the pipelines. + # + # Because Cloud Storage bucket names must be globally unique, the `name` + # specified below will be mapped to the unique bucket name: + # + # `{bucket_name_prefix}-{name}` + # + # Where the bucket name prefix is supplied using the `--bucket-name-prefix` + # parameter when running `scripts/generate_terraform.py`. + # + # Use hyphenated syntax, e.g. `some-prefix-123`, for the names. Note that + # bucket names must not contain "google" or close misspellings, such as + # "g00gle". + # + # Required Properties: + # name + # + # Optional Properties: + # location + # uniform_bucket_level_access (we suggest False for fine-grained access) + name: datacenter-cfe From b417ae38aa8fc9ecb270b6852543dd373d163d1a Mon Sep 17 00:00:00 2001 From: Adler Santos Date: Mon, 23 Aug 2021 11:10:49 -0400 Subject: [PATCH 2/5] YAML config revisions --- .../google_cfe/datacenter_cfe/pipeline.yaml | 61 +++---------------- 1 file changed, 10 insertions(+), 51 deletions(-) diff --git a/datasets/google_cfe/datacenter_cfe/pipeline.yaml b/datasets/google_cfe/datacenter_cfe/pipeline.yaml index 7474e19d4..2fbcddf09 100644 --- a/datasets/google_cfe/datacenter_cfe/pipeline.yaml +++ b/datasets/google_cfe/datacenter_cfe/pipeline.yaml @@ -23,56 +23,28 @@ resources: # We will keep adding to the list below to support more Google Cloud resources # over time. If a resource you need isn't supported, please file an issue on # the repository. - - type: bigquery_table # A Google BigQuery table to store your data. Requires a `bigquery_dataset` # to be specified in the config (i.e. `dataset.yaml) for the dataset that # this pipeline belongs in. - # - # Required Properties: - # table_id table_id: datacenter_cfe - # Optional Properties: - # Description of the table - description: ~ - dag: # [Required] Specify the Airflow version of the operators used by the DAG. airflow_version: 2 - # The DAG acronym stands for directed acyclic graph. This block represents - # your data pipeline along with every property and configuration it needs to - # onboard your data. initialize: - dag_id: PIPELINE_FOLDER_NAME + dag_id: datacenter_cfe default_args: owner: "Google" - - # When set to True, keeps a task from getting triggered if the previous schedule for the task hasn’t succeeded depends_on_past: False - start_date: '2021-03-01' + start_date: '2021-08-23' max_active_runs: 1 - schedule_interval: "@once" + schedule_interval: "@once" # The pipeline will be run manually once a year after new data is released catchup: False default_view: graph tasks: - # This is where you specify the tasks (a.k.a. processes) that your data - # pipeline will run to onboard the data. - # - # As the examples below will show, every task must be represented by an - # Airflow operator. The list of suported operators are listed in - # - # scripts/dag_imports.json - # - # If an operator you need isn't supported, please file an issue on the - # repository. - # - # Use the YAML list syntax in this block to specify every task for your - # pipeline. - - - operator: "GoogleCloudStorageToBigQueryOperator" # Initializes GCS to BQ task for the DAG. This operator is used to load a # JSON, CSV, Avro, ORC, or Parquet data from GCS into a BigQuery table. @@ -86,10 +58,10 @@ dag: task_id: "cfe_gcs_to_bq" # The GCS bucket where the CSV file is located in. - bucket: "{{ var.json.shared.composer_bucket }}" + bucket: "{{ var.json.google_cfe.source_bucket }}" - # The GCS object path for the CSV file - source_objects: ["data/DATASET_FOLDER_NAME/PIPELINE_FOLDER_NAME/run_date={{ ds }}/data.csv"] + # Use the CSV file containing data from 2019 up to the previous year + source_objects: ["data/2019-to-{{ macros.ds_format(macros.ds_add(ds, -366), '%Y-%m-%d', '%Y') }}.csv"] source_format: "CSV" destination_project_dataset_table: "google_cfe.datacenter_cfe" @@ -100,10 +72,6 @@ dag: # See https://cloud.google.com/bigquery/docs/reference/auditlogs/rest/Shared.Types/WriteDisposition write_disposition: "WRITE_TRUNCATE" - # The BigQuery table schema based on the CSV file. For more info, see - # https://cloud.google.com/bigquery/docs/schemas. - # Always use snake_case and lowercase for column names, and be explicit, - # i.e. specify modes for all columns. schema_fields: - name: "year" type: "INTEGER" @@ -124,20 +92,11 @@ dag: - name: "location" type: "STRING" mode: "REQUIRED" - description: "This is the "friendly name" of the Cloud Region. For Google Data Centers that are not Cloud regions, the location will be the country (non-US) or the state (US) that the Data Center is located in." + description: "This is the \"friendly name\" of the Cloud Region. For Google Data Centers that are not Cloud regions, the location will be the country (non-US) or the state (US) that the Data Center is located in." - name: "google_cfe" type: "FLOAT" mode: "NULLABLE" - description: "This metric is calculated for every hour in every region and tells us what percentage of the energy we consumed during an hour that is carbon-free. We take into account the carbon-free energy that’s already supplied by the grid, in addition to the investments we have made in renewable energy in that location to reach our 24/7 carbon-free objective (https://www.gstatic.com/gumdrop/sustainability/247-carbon-free-energy.pdf). We then aggregate the available average hourly CFE percentage for each region for the year. We do not currently have the hourly energy information available for calculating the metrics for all regions. We anticipate rolling out the calculated metrics using hourly data to regions as the data becomes available." + description: "This metric is calculated for every hour in every region and tells us what percentage of the energy we consumed during an hour that is carbon-free. We take into account the carbon-free energy that's already supplied by the grid, in addition to the investments we have made in renewable energy in that location to reach our 24/7 carbon-free objective (https://www.gstatic.com/gumdrop/sustainability/247-carbon-free-energy.pdf). We then aggregate the available average hourly CFE percentage for each region for the year. We do not currently have the hourly energy information available for calculating the metrics for all regions. We anticipate rolling out the calculated metrics using hourly data to regions as the data becomes available." - # Initializes a GCS-to-GCS task for the DAG. This operator is used to copy or move - # GCS objects from one location to another. - - graph_paths: - # This is where you specify the relationships (i.e. directed paths/edges) - # among the tasks specified above. Use the bitshift operator to define the - # relationships and the `task_id` value above to represent tasks. - # - # For more info, see - # https://airflow.apache.org/docs/apache-airflow/stable/tutorial.html#setting-up-dependencies - - "cfe_gcs_to_bq" \ No newline at end of file + graph_paths: + - "cfe_gcs_to_bq" From f62db3a1f41265ad074b1efbc8dddea60ecce944 Mon Sep 17 00:00:00 2001 From: Adler Santos Date: Mon, 23 Aug 2021 11:11:20 -0400 Subject: [PATCH 3/5] generate Terraform files from YAML config --- .../_terraform/datacenter_cfe_pipeline.tf | 39 +++++++++++++++++++ .../_terraform/google_cfe_dataset.tf | 34 ++++++++++++++++ datasets/google_cfe/_terraform/provider.tf | 28 +++++++++++++ datasets/google_cfe/_terraform/variables.tf | 23 +++++++++++ 4 files changed, 124 insertions(+) create mode 100644 datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf create mode 100644 datasets/google_cfe/_terraform/google_cfe_dataset.tf create mode 100644 datasets/google_cfe/_terraform/provider.tf create mode 100644 datasets/google_cfe/_terraform/variables.tf diff --git a/datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf b/datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf new file mode 100644 index 000000000..92ddd6b6e --- /dev/null +++ b/datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +resource "google_bigquery_table" "datacenter_cfe" { + project = var.project_id + dataset_id = "google_cfe" + table_id = "datacenter_cfe" + + + + + + + depends_on = [ + google_bigquery_dataset.google_cfe + ] +} + +output "bigquery_table-datacenter_cfe-table_id" { + value = google_bigquery_table.datacenter_cfe.table_id +} + +output "bigquery_table-datacenter_cfe-id" { + value = google_bigquery_table.datacenter_cfe.id +} diff --git a/datasets/google_cfe/_terraform/google_cfe_dataset.tf b/datasets/google_cfe/_terraform/google_cfe_dataset.tf new file mode 100644 index 000000000..7fa17d953 --- /dev/null +++ b/datasets/google_cfe/_terraform/google_cfe_dataset.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +resource "google_bigquery_dataset" "google_cfe" { + dataset_id = "google_cfe" + project = var.project_id +} + +output "bigquery_dataset-google_cfe-dataset_id" { + value = google_bigquery_dataset.google_cfe.dataset_id +} + +resource "google_storage_bucket" "datacenter-cfe" { + name = "${var.bucket_name_prefix}-datacenter-cfe" + force_destroy = true +} + +output "storage_bucket-datacenter-cfe-name" { + value = google_storage_bucket.datacenter-cfe.name +} diff --git a/datasets/google_cfe/_terraform/provider.tf b/datasets/google_cfe/_terraform/provider.tf new file mode 100644 index 000000000..23ab87dcd --- /dev/null +++ b/datasets/google_cfe/_terraform/provider.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +provider "google" { + project = var.project_id + impersonate_service_account = var.impersonating_acct + region = var.region +} + +data "google_client_openid_userinfo" "me" {} + +output "impersonating-account" { + value = data.google_client_openid_userinfo.me.email +} diff --git a/datasets/google_cfe/_terraform/variables.tf b/datasets/google_cfe/_terraform/variables.tf new file mode 100644 index 000000000..c3ec7c506 --- /dev/null +++ b/datasets/google_cfe/_terraform/variables.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +variable "project_id" {} +variable "bucket_name_prefix" {} +variable "impersonating_acct" {} +variable "region" {} +variable "env" {} + From b452b48985ddd8a142f655b4e28eb73ea2805955 Mon Sep 17 00:00:00 2001 From: Adler Santos Date: Mon, 23 Aug 2021 11:11:31 -0400 Subject: [PATCH 4/5] generate DAG file from YAML config --- .../datacenter_cfe/datacenter_cfe_dag.py | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 datasets/google_cfe/datacenter_cfe/datacenter_cfe_dag.py diff --git a/datasets/google_cfe/datacenter_cfe/datacenter_cfe_dag.py b/datasets/google_cfe/datacenter_cfe/datacenter_cfe_dag.py new file mode 100644 index 000000000..d7761e7fc --- /dev/null +++ b/datasets/google_cfe/datacenter_cfe/datacenter_cfe_dag.py @@ -0,0 +1,86 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from airflow import DAG +from airflow.providers.google.cloud.transfers import gcs_to_bigquery + +default_args = { + "owner": "Google", + "depends_on_past": False, + "start_date": "2021-08-23", +} + + +with DAG( + dag_id="google_cfe.datacenter_cfe", + default_args=default_args, + max_active_runs=1, + schedule_interval="@once", + catchup=False, + default_view="graph", +) as dag: + + # Task to load CSV data to a BigQuery table + cfe_gcs_to_bq = gcs_to_bigquery.GCSToBigQueryOperator( + task_id="cfe_gcs_to_bq", + bucket="{{ var.json.google_cfe.source_bucket }}", + source_objects=[ + "data/2019-to-{{ macros.ds_format(macros.ds_add(ds, -366), '%Y-%m-%d', '%Y') }}.csv" + ], + source_format="CSV", + destination_project_dataset_table="google_cfe.datacenter_cfe", + skip_leading_rows=1, + write_disposition="WRITE_TRUNCATE", + schema_fields=[ + { + "name": "year", + "type": "INTEGER", + "mode": "REQUIRED", + "description": "The year for which the Google CFE metric has been aggregated. We will continue to add new data for each year as we make progress. Note that the Google CFE metric will only be included for the years that the Cloud Region or Data Center was operational. For example, the Data Center in Denmark came online in 2020, so the Google CFE data for that region starts in 2020 (and there is no data for Denmark for 2019).", + }, + { + "name": "cfe_region", + "type": "STRING", + "mode": "REQUIRED", + "description": "The regional boundary of the electric grid we consider when calculating the Google CFE score. For most of the world, the CFE region is defined as the country. However, for countries that have distinct grids, such as the US, there are multiple CFE regions, which are defined by the balancing authority.", + }, + { + "name": "zone_id", + "type": "STRING", + "mode": "REQUIRED", + "description": "This is the ID associated with the CFE Region based on Tomorrow's ElectricityMap API definition. (http://static.electricitymap.org/api/docs/index.html#zones)", + }, + { + "name": "cloud_region", + "type": "STRING", + "mode": "REQUIRED", + "description": "The Google Cloud Region that is mapped to the CFE region. For Google Data Centers that are not Cloud Regions, the region will be labeled 'non-cloud-data-center'.", + }, + { + "name": "location", + "type": "STRING", + "mode": "REQUIRED", + "description": 'This is the "friendly name" of the Cloud Region. For Google Data Centers that are not Cloud regions, the location will be the country (non-US) or the state (US) that the Data Center is located in.', + }, + { + "name": "google_cfe", + "type": "FLOAT", + "mode": "NULLABLE", + "description": "This metric is calculated for every hour in every region and tells us what percentage of the energy we consumed during an hour that is carbon-free. We take into account the carbon-free energy that's already supplied by the grid, in addition to the investments we have made in renewable energy in that location to reach our 24/7 carbon-free objective (https://www.gstatic.com/gumdrop/sustainability/247-carbon-free-energy.pdf). We then aggregate the available average hourly CFE percentage for each region for the year. We do not currently have the hourly energy information available for calculating the metrics for all regions. We anticipate rolling out the calculated metrics using hourly data to regions as the data becomes available.", + }, + ], + ) + + cfe_gcs_to_bq From 25a6c962cb659772b0db33544c5d698859ff8e06 Mon Sep 17 00:00:00 2001 From: Adler Santos Date: Thu, 26 Aug 2021 17:11:26 -0400 Subject: [PATCH 5/5] add BQ descriptions for CFE definition --- datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf | 2 +- datasets/google_cfe/_terraform/google_cfe_dataset.tf | 5 +++-- datasets/google_cfe/datacenter_cfe/pipeline.yaml | 1 + datasets/google_cfe/dataset.yaml | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf b/datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf index 92ddd6b6e..4b9297e06 100644 --- a/datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf +++ b/datasets/google_cfe/_terraform/datacenter_cfe_pipeline.tf @@ -20,7 +20,7 @@ resource "google_bigquery_table" "datacenter_cfe" { dataset_id = "google_cfe" table_id = "datacenter_cfe" - + description = "Carbon-free energy (CFE) scores for Google Cloud regions and other Google data center regions" diff --git a/datasets/google_cfe/_terraform/google_cfe_dataset.tf b/datasets/google_cfe/_terraform/google_cfe_dataset.tf index 7fa17d953..2ccf19df3 100644 --- a/datasets/google_cfe/_terraform/google_cfe_dataset.tf +++ b/datasets/google_cfe/_terraform/google_cfe_dataset.tf @@ -16,8 +16,9 @@ resource "google_bigquery_dataset" "google_cfe" { - dataset_id = "google_cfe" - project = var.project_id + dataset_id = "google_cfe" + project = var.project_id + description = "Carbon-free energy (CFE) scores for Google Cloud regions and other Google data center regions" } output "bigquery_dataset-google_cfe-dataset_id" { diff --git a/datasets/google_cfe/datacenter_cfe/pipeline.yaml b/datasets/google_cfe/datacenter_cfe/pipeline.yaml index 2fbcddf09..2abe38426 100644 --- a/datasets/google_cfe/datacenter_cfe/pipeline.yaml +++ b/datasets/google_cfe/datacenter_cfe/pipeline.yaml @@ -28,6 +28,7 @@ resources: # to be specified in the config (i.e. `dataset.yaml) for the dataset that # this pipeline belongs in. table_id: datacenter_cfe + description: "Carbon-free energy (CFE) scores for Google Cloud regions and other Google data center regions" dag: # [Required] Specify the Airflow version of the operators used by the DAG. diff --git a/datasets/google_cfe/dataset.yaml b/datasets/google_cfe/dataset.yaml index ec778d8d4..22a6da438 100644 --- a/datasets/google_cfe/dataset.yaml +++ b/datasets/google_cfe/dataset.yaml @@ -55,7 +55,7 @@ resources: # description (A user-friendly description of the dataset) # location (The geographic location where the dataset should reside) dataset_id: google_cfe - description: ~ + description: "Carbon-free energy (CFE) scores for Google Cloud regions and other Google data center regions" - type: storage_bucket # Google Cloud Storage Bucket that your pipelines need. Say, you need an