feat: Support Terraform remote state when generating GCP resources (#39)

README.md

@@ -82,6 +82,8 @@ $ python scripts/generate_terraform.py \
     --region REGION \
     --bucket-name-prefix UNIQUE_BUCKET_PREFIX \
     [--env] dev \
+    [--tf-state-bucket] \
+    [--tf-state-prefix] \
     [--tf-apply] \
     [--impersonating-acct] IMPERSONATING_SERVICE_ACCT
 ```
@@ -90,6 +92,8 @@ This generates Terraform files (`*.tf`) in a `_terraform` directory inside that
 
 The `--bucket-name-prefix` is used to ensure that the buckets created by different environments and contributors are kept unique. This is to satisfy the rule where bucket names must be globally unique across all of GCS. Use hyphenated names (`some-prefix-123`) instead of snakecase or underscores (`some_prefix_123`).
 
+The `--tf-state-bucket` and `--tf-state-prefix` parameters can be optionally used if one needs to use a remote store for the Terraform state. This will create a `backend.tf` file that points to the GCS bucket and prefix to use in storing the Terraform state. For more info, see the [Terraform docs for using GCS backends](https://www.terraform.io/docs/language/settings/backends/gcs.html).
+
 In addition, the command above creates a "dot" directory in the project root. The directory name is the value you pass to the `--env` parameter of the command. If no `--env` argument was passed, the value defaults to `dev` (which generates the `.dev` folder).
 
 Consider this "dot" directory as your own dedicated space for prototyping. The files and variables created in that directory will use an isolated environment. All such directories are gitignored.

datasets/ml_datasets/_terraform/ml_datasets_dataset.tf

@@ -18,7 +18,7 @@
 resource "google_bigquery_dataset" "ml_datasets" {
   dataset_id = "ml_datasets"
   project    = var.project_id
-  }
+}
 
 output "bigquery_dataset-ml_datasets-dataset_id" {
   value = google_bigquery_dataset.ml_datasets.dataset_id

datasets/ml_datasets/_terraform/penguins_pipeline.tf

@@ -20,7 +20,7 @@ resource "google_bigquery_table" "penguins" {
   dataset_id = "ml_datasets"
   table_id   = "penguins"
 
-  
+
 
   depends_on = [
     google_bigquery_dataset.ml_datasets

datasets/ml_datasets/_terraform/provider.tf

@@ -16,9 +16,10 @@
 
 
 provider "google" {
-  project = var.project_id
-  region  = var.region
-  }
+  project                     = var.project_id
+  impersonate_service_account = var.impersonating_acct
+  region                      = var.region
+}
 
 data "google_client_openid_userinfo" "me" {}
 

datasets/ml_datasets/_terraform/variables.tf

@@ -16,7 +16,7 @@
 
 
 variable "project_id" {}
-variable "project_num" {}
+variable "bucket_name_prefix" {}
 variable "impersonating_acct" {}
 variable "region" {}
 variable "env" {}

scripts/generate_terraform.py

@@ -35,6 +35,7 @@
     "tfvars": TEMPLATES_PATH / "terraform.tfvars.jinja2",
     "variables": TEMPLATES_PATH / "variables.tf.jinja2",
     "provider": TEMPLATES_PATH / "provider.tf.jinja2",
+    "backend": TEMPLATES_PATH / "backend.tf.jinja2",
 }
 
 yaml = yaml.YAML(typ="safe")
@@ -47,6 +48,8 @@ def main(
     region: str,
     impersonating_acct: str,
     env: str,
+    tf_state_bucket: str,
+    tf_state_prefix: str,
     tf_apply: bool = False,
 ):
     validate_bucket_name(bucket_name_prefix)
@@ -55,6 +58,7 @@ def main(
     create_gitignored_env_path(dataset_id, env_path)
 
     generate_provider_tf(project_id, dataset_id, region, impersonating_acct, env_path)
+    generate_backend_tf(dataset_id, tf_state_bucket, tf_state_prefix, env_path)
 
     dataset_config = yaml.load(open(DATASETS_PATH / dataset_id / "dataset.yaml"))
     generate_dataset_tf(dataset_id, project_id, dataset_config, env)
@@ -86,7 +90,26 @@ def generate_provider_tf(
         },
     )
 
-    create_file_in_dot_and_project_dirs(dataset_id, contents, "provider.tf", env_path)
+    create_file_in_dir_tree(dataset_id, contents, "provider.tf", env_path)
+
+
+def generate_backend_tf(
+    dataset_id: str, tf_state_bucket: str, tf_state_prefix: str, env_path: pathlib.Path
+):
+    if not tf_state_bucket:
+        return
+
+    contents = apply_substitutions_to_template(
+        TEMPLATE_PATHS["backend"],
+        {
+            "tf_state_bucket": tf_state_bucket,
+            "tf_state_prefix": tf_state_prefix,
+        },
+    )
+
+    create_file_in_dir_tree(
+        dataset_id, contents, "backend.tf", env_path, use_project_dir=False
+    )
 
 
 def generate_dataset_tf(dataset_id: str, project_id: str, config: dict, env: str):
@@ -99,7 +122,7 @@ def generate_dataset_tf(dataset_id: str, project_id: str, config: dict, env: str
     for resource in config["resources"]:
         contents += tf_resource_contents(resource, copy.deepcopy(subs))
 
-    create_file_in_dot_and_project_dirs(
+    create_file_in_dir_tree(
         dataset_id, contents, f"{dataset_id}_dataset.tf", PROJECT_ROOT / f".{env}"
     )
 
@@ -130,14 +153,14 @@ def generate_pipeline_tf(
     for resource in config["resources"]:
         contents += tf_resource_contents(resource, {**subs, **resource})
 
-    create_file_in_dot_and_project_dirs(
+    create_file_in_dir_tree(
         dataset_id, contents, f"{pipeline_id}_pipeline.tf", env_path
     )
 
 
 def generate_variables_tf(dataset_id: str, env_path: pathlib.Path):
     contents = pathlib.Path(TEMPLATE_PATHS["variables"]).read_text()
-    create_file_in_dot_and_project_dirs(dataset_id, contents, "variables.tf", env_path)
+    create_file_in_dir_tree(dataset_id, contents, "variables.tf", env_path)
 
 
 def generate_tfvars_file(
@@ -219,14 +242,24 @@ def create_gitignored_env_path(dataset_id: str, env_path: pathlib.Path):
     (env_path / "datasets" / dataset_id).mkdir(parents=True, exist_ok=True)
 
 
-def create_file_in_dot_and_project_dirs(
-    dataset_id: str, contents: str, filename: str, env_path: pathlib.Path
+def create_file_in_dir_tree(
+    dataset_id: str,
+    contents: str,
+    filename: str,
+    env_path: pathlib.Path,
+    use_env_dir: bool = True,
+    use_project_dir: bool = True,
 ):
     filepaths = []
-    for prefix in (
-        env_path / "datasets" / dataset_id / "_terraform",
-        DATASETS_PATH / dataset_id / "_terraform",
-    ):
+    prefixes = []
+
+    if use_env_dir:
+        prefixes.append(env_path / "datasets" / dataset_id / "_terraform")
+
+    if use_project_dir:
+        prefixes.append(DATASETS_PATH / dataset_id / "_terraform")
+
+    for prefix in prefixes:
         if not prefix.exists():
             prefix.mkdir(parents=True, exist_ok=True)
 
@@ -303,6 +336,19 @@ def apply_substitutions_to_template(template: pathlib.Path, subs: dict) -> str:
         dest="bucket_name_prefix",
         help="The prefix to use for GCS bucket names for global uniqueness",
     )
+    parser.add_argument(
+        "--tf-state-bucket",
+        type=str,
+        dest="tf_state_bucket",
+        help="The GCS bucket name for the Terraform remote state",
+    )
+    parser.add_argument(
+        "--tf-state-prefix",
+        type=str,
+        default="terraform/state",
+        dest="tf_state_prefix",
+        help="The GCS bucket prefix for the Terraform remote state",
+    )
     parser.add_argument(
         "-e",
         "--env",
@@ -335,5 +381,7 @@ def apply_substitutions_to_template(template: pathlib.Path, subs: dict) -> str:
         args.region,
         args.impersonating_acct,
         args.env,
+        args.tf_state_bucket,
+        args.tf_state_prefix,
         args.tf_apply,
     )

templates/terraform/backend.tf.jinja2

@@ -0,0 +1,23 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+terraform {
+  backend "gcs" {
+    bucket = "{{ tf_state_bucket }}"
+    prefix = "{{ tf_state_prefix }}"
+  }
+}