feat: Add terraform validator and add policy-library (#263)

Co-authored-by: Amanda Karina Lopes de Oliveira <amandak@ciandt.com>
Co-authored-by: Amanda Karina Lopes de Oliveira <55760933+amandakarina@users.noreply.github.com>

0-bootstrap/README.md

@@ -37,6 +37,10 @@ If you are using the `jenkins_bootstrap` sub-module, please see [README-Jenkins]
 1. Copy tfvars by running `cp terraform.example.tfvars terraform.tfvars` and update `terraform.tfvars` with values from your environment.
 1. Run `terraform init`
 1. Run `terraform plan` and review output
+1. To run terraform-validator steps please follow these [instructions](https://github.com/forseti-security/policy-library/blob/master/docs/user_guide.md#install-terraform-validator) and install the latest version.
+    1. Run `terraform plan -input=false -out bootstrap.tfplan`
+    1. Run `terraform show -json bootstrap.tfplan > bootstrap.json`
+    1. Run `terraform-validator validate bootstrap.json --policy-path="../policy-library" --project <A-VALID-PROJECT>` and check for violations.
 1. Run `terraform apply`
 1. Run `terraform output gcs_bucket_tfstate` to get your GCS bucket from the apply step
 1. Copy the backend by running `cp backend.tf.example backend.tf` and update `backend.tf` with your GCS bucket.
@@ -56,7 +60,7 @@ Currently, the bucket information is replaced in the state backends as a part of
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | billing\_account | The ID of the billing account to associate projects with. | `string` | n/a | yes |
-| cloud\_source\_repos | List of Cloud Source Reposiories created during bootrstap project build stage | `list(string)` | <pre>[<br>  "gcp-org",<br>  "gcp-environments",<br>  "gcp-networks",<br>  "gcp-projects"<br>]</pre> | no |
+| cloud\_source\_repos | List of Cloud Source Reposiories created during bootrstap project build stage | `list(string)` | <pre>[<br>  "gcp-org",<br>  "gcp-environments",<br>  "gcp-networks",<br>  "gcp-projects",<br>  "gcp-policies"<br>]</pre> | no |
 | default\_region | Default region to create resources where applicable. | `string` | `"us-central1"` | no |
 | folder\_prefix | Name prefix to use for folders created. | `string` | `"fldr"` | no |
 | group\_billing\_admins | Google Group for GCP Billing Administrators | `string` | n/a | yes |

0-bootstrap/main.tf

@@ -113,21 +113,24 @@ resource "google_billing_account_iam_member" "tf_billing_admin" {
 
 // Comment-out the cloudbuild_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build
 module "cloudbuild_bootstrap" {
-  source                    = "terraform-google-modules/bootstrap/google//modules/cloudbuild"
-  version                   = "~> 1.3"
-  org_id                    = var.org_id
-  folder_id                 = google_folder.bootstrap.id
-  billing_account           = var.billing_account
-  group_org_admins          = var.group_org_admins
-  default_region            = var.default_region
-  terraform_sa_email        = module.seed_bootstrap.terraform_sa_email
-  terraform_sa_name         = module.seed_bootstrap.terraform_sa_name
-  terraform_state_bucket    = module.seed_bootstrap.gcs_bucket_tfstate
-  sa_enable_impersonation   = true
-  cloudbuild_plan_filename  = "cloudbuild-tf-plan.yaml"
-  cloudbuild_apply_filename = "cloudbuild-tf-apply.yaml"
-  project_prefix            = var.project_prefix
-  cloud_source_repos        = var.cloud_source_repos
+  source                      = "terraform-google-modules/bootstrap/google//modules/cloudbuild"
+  version                     = "~> 1.3"
+  org_id                      = var.org_id
+  folder_id                   = google_folder.bootstrap.id
+  billing_account             = var.billing_account
+  group_org_admins            = var.group_org_admins
+  default_region              = var.default_region
+  terraform_sa_email          = module.seed_bootstrap.terraform_sa_email
+  terraform_sa_name           = module.seed_bootstrap.terraform_sa_name
+  terraform_state_bucket      = module.seed_bootstrap.gcs_bucket_tfstate
+  sa_enable_impersonation     = true
+  cloudbuild_plan_filename    = "cloudbuild-tf-plan.yaml"
+  cloudbuild_apply_filename   = "cloudbuild-tf-apply.yaml"
+  project_prefix              = var.project_prefix
+  cloud_source_repos          = var.cloud_source_repos
+  terraform_validator_release = "2021-01-21"
+  terraform_version           = "0.13.5"
+  terraform_version_sha256sum = "f7b7a7b1bfbf5d78151cfe3d1d463140b5fd6a354e71a7de2b5644e652ca5147"
 
   activate_apis = [
     "serviceusage.googleapis.com",
@@ -161,6 +164,34 @@ module "cloudbuild_bootstrap" {
   ]
 }
 
+resource "google_project_iam_member" "project_source_reader" {
+  project = module.cloudbuild_bootstrap.cloudbuild_project_id
+  role    = "roles/source.reader"
+  member  = "serviceAccount:${module.seed_bootstrap.terraform_sa_email}"
+
+  depends_on = [module.cloudbuild_bootstrap.csr_repos]
+}
+
+data "google_project" "cloudbuild" {
+  project_id = module.cloudbuild_bootstrap.cloudbuild_project_id
+
+  depends_on = [module.cloudbuild_bootstrap.csr_repos]
+}
+
+resource "google_organization_iam_member" "org_cb_sa_browser" {
+  count  = var.parent_folder == "" ? 1 : 0
+  org_id = var.org_id
+  role   = "roles/browser"
+  member = "serviceAccount:${data.google_project.cloudbuild.number}@cloudbuild.gserviceaccount.com"
+}
+
+resource "google_folder_iam_member" "folder_cb_sa_browser" {
+  count  = var.parent_folder != "" ? 1 : 0
+  folder = var.parent_folder
+  role   = "roles/browser"
+  member = "serviceAccount:${data.google_project.cloudbuild.number}@cloudbuild.gserviceaccount.com"
+}
+
 ## Un-comment the jenkins_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build
 # module "jenkins_bootstrap" {
 #  source                                  = "./modules/jenkins-agent"
@@ -189,3 +220,17 @@ module "cloudbuild_bootstrap" {
 #  tunnel1_bgp_peer_address                = var.tunnel1_bgp_peer_address
 #  tunnel1_bgp_session_range               = var.tunnel1_bgp_session_range
 # }
+
+# resource "google_organization_iam_member" "org_jenkins_sa_browser" {
+#   count  = var.parent_folder == "" ? 1 : 0
+#   org_id = var.org_id
+#   role   = "roles/browser"
+#   member = "serviceAccount:${module.jenkins_bootstrap.jenkins_agent_sa_email}"
+# }
+
+# resource "google_folder_iam_member" "folder_jenkins_sa_browser" {
+#   count  = var.parent_folder != "" ? 1 : 0
+#   folder = var.parent_folder
+#   role   = "roles/browser"
+#   member = "serviceAccount:${module.jenkins_bootstrap.jenkins_agent_sa_email}"
+# }

0-bootstrap/modules/jenkins-agent/files/jenkins_gce_startup_script.sh

@@ -42,7 +42,7 @@ wget "https://releases.hashicorp.com/terraform/${tpl_TERRAFORM_VERSION}/terrafor
     rm -rf /var/lib/apt/lists/*
 
 echo "**** Startup Step 6/9: Download and install the Terraform validator ****"
-gsutil cp gs://terraform-validator/releases/2020-03-05/terraform-validator-linux-amd64 .
+gsutil cp gs://terraform-validator/releases/2021-01-21/terraform-validator-linux-amd64 .
 chmod 755 "${tpl_TERRAFORM_DIR}terraform-validator-linux-amd64"
 mv "${tpl_TERRAFORM_DIR}terraform-validator-linux-amd64" "${tpl_TERRAFORM_DIR}terraform-validator"
 

0-bootstrap/variables.tf

@@ -72,7 +72,7 @@ variable "folder_prefix" {
 variable "cloud_source_repos" {
   description = "List of Cloud Source Reposiories created during bootrstap project build stage"
   type        = list(string)
-  default     = ["gcp-bootstrap", "gcp-org", "gcp-environments", "gcp-networks", "gcp-projects"]
+  default     = ["gcp-org", "gcp-environments", "gcp-networks", "gcp-projects", "gcp-policies"]
 }
 
 /* ----------------------------------------

1-org/README.md

@@ -1,6 +1,6 @@
 # 1-org
 
-The purpose of this step is to set up top level shared folders, monitoring & networking projects, org level logging and set baseline security settings through organizational policy.
+The purpose of this step is to set up top level shared folders, monitoring & networking projects, org level logging and set baseline security settings through organizational policy. This step also creates the shared repository that host the terraform validator policies.
 
 ## Prerequisites
 
@@ -26,6 +26,13 @@ OS Login has some [limitations](https://cloud.google.com/compute/docs/instances/
 If those limitations do not apply to your workload/environment you can choose to enable the OS Login policy by setting variable `enable_os_login_policy` to `true`.
 
 ### Setup to run via Cloud Build
+
+1. Clone repo `gcloud source repos clone gcp-policies --project=YOUR_CLOUD_BUILD_PROJECT_ID`.
+1. Navigate into the repo `cd gcp-policies`.
+1. Copy contents of policy-library to new repo `cp -RT ../terraform-example-foundation/policy-library/ .` (modify accordingly based on your current directory).
+1. Commit changes with `git add .` and `git commit -m 'Your message'`
+1. Push your master branch to the new repo `git push --set-upstream origin master`.
+1. Navigate out of the repo `cd ..`.
 1. Clone repo `gcloud source repos clone gcp-org --project=YOUR_CLOUD_BUILD_PROJECT_ID` (this is from terraform output from the previous section, 0-bootstrap).
 1. Navigate into the repo `cd gcp-org` and change to a non production branch `git checkout -b plan`
 1. Copy contents of foundation to new repo `cp -RT ../terraform-example-foundation/1-org/ .` (modify accordingly based on your current directory).
@@ -41,15 +48,17 @@ If those limitations do not apply to your workload/environment you can choose to
     1. Review the apply output in your cloud build project https://console.cloud.google.com/cloud-build/builds?project=YOUR_CLOUD_BUILD_PROJECT_ID
 
 ### Setup to run via Jenkins
+
 1. Clone the repo you created manually in bootstrap: `git clone <YOUR_NEW_REPO-1-org>`
 1. Navigate into the repo `cd YOUR_NEW_REPO_CLONE-1-org` and change to a non production branch `git checkout -b plan`
 1. Copy contents of foundation to new repo `cp -RT ../terraform-example-foundation/1-org/ .` (modify accordingly based on your current directory).
+1. Copy contents of policy-library to new repo `cp -RT ../terraform-example-foundation/policy-library/ ./policy-library` (modify accordingly based on your current directory).
 1. Copy the Jenkinsfile script `cp ../terraform-example-foundation/build/Jenkinsfile .` to the root of your new repository (modify accordingly based on your current directory).
 1. Update the variables located in the `environment {}` section of the `Jenkinsfile` with values from your environment:
     ```
-    _POLICY_REPO (optional)
     _TF_SA_EMAIL
     _STATE_BUCKET_NAME
+    _PROJECT_ID (the cicd project id)
     ```
 1. Copy terraform wrapper script `cp ../terraform-example-foundation/build/tf-wrapper.sh . ` to the root of your new repository (modify accordingly based on your current directory).
 1. Ensure wrapper script can be executed `chmod 755 ./tf-wrapper.sh`.
@@ -65,6 +74,7 @@ If those limitations do not apply to your workload/environment you can choose to
 1. You can now move to the instructions in the step [2-environments](../2-environments/README.md).
 
 ### Run terraform locally
+
 1. Change into 1-org folder.
 1. Run `cp ../build/tf-wrapper.sh .`
 1. Run `chmod 755 ./tf-wrapper.sh`
@@ -77,8 +87,11 @@ You can run `terraform output gcs_bucket_tfstate` in the 0-bootstap folder to ob
 We will now deploy our environment (production) using this script.
 When using Cloud Build or Jenkins as your CI/CD tool each environment corresponds to a branch is the repository for 1-org step and only the corresponding environment is applied.
 
+To use the `validate` option of the `tf-wrapper.sh` script, the latest version of `terraform-validator` must be [installed](https://github.com/forseti-security/policy-library/blob/master/docs/user_guide.md#how-to-use-terraform-validator) in your system and in you `PATH`.
+
 1. Run `./tf-wrapper.sh init production`
 1. Run `./tf-wrapper.sh plan production` and review output.
+1. Run `./tf-wrapper.sh validate production $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply production`
 
 If you received any errors or made any changes to the Terraform config or `terraform.tfvars` you must re-run `./tf-wrapper.sh plan production` before run `./tf-wrapper.sh apply production`

2-environments/README.md

@@ -36,9 +36,9 @@ The purpose of this step is to set up development, non-production and production
 1. Copy the Jenkinsfile script `cp ../terraform-example-foundation/build/Jenkinsfile .` to the root of your new repository (modify accordingly based on your current directory).
 1. Update the variables located in the `environment {}` section of the `Jenkinsfile` with values from your environment:
     ```
-    _POLICY_REPO (optional)
     _TF_SA_EMAIL
     _STATE_BUCKET_NAME
+    _PROJECT_ID (the cicd project id)
     ```
 1. Copy terraform wrapper script `cp ../terraform-example-foundation/build/tf-wrapper.sh . ` to the root of your new repository (modify accordingly based on your current directory).
 1. Ensure wrapper script can be executed `chmod 755 ./tf-wrapper.sh`.
@@ -68,14 +68,19 @@ You can run `terraform output gcs_bucket_tfstate` in the 0-bootstap folder to ob
 We will now deploy each of our environments(development/production/non-production) using this script.
 When using Cloud Build or Jenkins as your CI/CD tool each environment corresponds to a branch is the repository for 2-environments step and only the corresponding environment is applied.
 
+To use the `validate` option of the `tf-wrapper.sh` script, the latest version of `terraform-validator` must be [installed](https://github.com/forseti-security/policy-library/blob/master/docs/user_guide.md#how-to-use-terraform-validator) in your system and in you `PATH`.
+
 1. Run `./tf-wrapper.sh init development`
 1. Run `./tf-wrapper.sh plan development` and review output.
+1. Run `./tf-wrapper.sh validate development $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply development`
 1. Run `./tf-wrapper.sh init non-production`
 1. Run `./tf-wrapper.sh plan non-production` and review output.
+1. Run `./tf-wrapper.sh validate non-production $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply non-production`
 1. Run `./tf-wrapper.sh init production`
 1. Run `./tf-wrapper.sh plan production` and review output.
+1. Run `./tf-wrapper.sh validate production $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply production`
 
 If you received any errors or made any changes to the Terraform config or `terraform.tfvars` you must re-run `./tf-wrapper.sh plan <env>` before run `./tf-wrapper.sh apply <env>`

3-networks/README.md

@@ -68,9 +68,9 @@ If you are not able to use dedicated interconnect, you can also use an HA VPN to
 1. Copy the Jenkinsfile script `cp ../terraform-example-foundation/build/Jenkinsfile .` to the root of your new repository (modify accordingly based on your current directory).
 1. Update the variables located in the `environment {}` section of the `Jenkinsfile` with values from your environment:
     ```
-    _POLICY_REPO (optional)
     _TF_SA_EMAIL
     _STATE_BUCKET_NAME
+    _PROJECT_ID (the cicd project id)
     ```
 1. Copy terraform wrapper script `cp ../terraform-example-foundation/build/tf-wrapper.sh . ` to the root of your new repository (modify accordingly based on your current directory).
 1. Ensure wrapper script can be executed `chmod 755 ./tf-wrapper.sh`.
@@ -114,17 +114,23 @@ We will now deploy each of our environments(development/production/non-productio
 When using Cloud Build or Jenkins as your CI/CD tool each environment corresponds to a branch in the repository for 3-networks step
 and only the corresponding environment is applied.
 
+To use the `validate` option of the `tf-wrapper.sh` script, the latest version of `terraform-validator` must be [installed](https://github.com/forseti-security/policy-library/blob/master/docs/user_guide.md#how-to-use-terraform-validator) in your system and in you `PATH`.
+
 1. Run `./tf-wrapper.sh init shared`
 1. Run `./tf-wrapper.sh plan shared` and review output.
+1. Run `./tf-wrapper.sh validate shared $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply shared`
 1. Run `./tf-wrapper.sh init production`
 1. Run `./tf-wrapper.sh plan production` and review output.
+1. Run `./tf-wrapper.sh validate production $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply production`
 1. Run `./tf-wrapper.sh init non-production`
 1. Run `./tf-wrapper.sh plan non-production` and review output.
+1. Run `./tf-wrapper.sh validate non-production $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply non-production`
 1. Run `./tf-wrapper.sh init development`
 1. Run `./tf-wrapper.sh plan development` and review output.
+1. Run `./tf-wrapper.sh validate development $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply development`
 
 If you received any errors or made any changes to the Terraform config or any `.tfvars`you must re-run `./tf-wrapper.sh plan <env>` before run `./tf-wrapper.sh apply <env>`

4-projects/README.md

@@ -44,9 +44,9 @@ If your user does not have access to run the commands above and you are in the o
 1. Copy the Jenkinsfile script `cp ../terraform-example-foundation/build/Jenkinsfile .` to the root of your new repository (modify accordingly based on your current directory).
 1. Update the variables located in the `environment {}` section of the `Jenkinsfile` with values from your environment:
     ```
-    _POLICY_REPO (optional)
     _TF_SA_EMAIL
     _STATE_BUCKET_NAME
+    _PROJECT_ID (the cicd project id)
     ```
 1. Copy terraform wrapper script `cp ../terraform-example-foundation/build/tf-wrapper.sh . ` to the root of your new repository (modify accordingly based on your current directory).
 1. Ensure wrapper script can be executed `chmod 755 ./tf-wrapper.sh`.
@@ -83,14 +83,20 @@ You can run `terraform output gcs_bucket_tfstate` in the 0-bootstap folder to ob
 We will now deploy each of our environments(development/production/non-production) using this script.
 When using Cloud Build or Jenkins as your CI/CD tool each environment corresponds to a branch is the repository for 4-projects step and only the corresponding environment is applied.
 
+To use the `validate` option of the `tf-wrapper.sh` script, the latest version of `terraform-validator` must be [installed](https://github.com/forseti-security/policy-library/blob/master/docs/user_guide.md#how-to-use-terraform-validator) in your system and in you `PATH`.
+
 1. Run `./tf-wrapper.sh init production`
 1. Run `./tf-wrapper.sh plan production` and review output.
+1. Run `./tf-wrapper.sh validate production $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply production`
 1. Run `./tf-wrapper.sh init non-production`
 1. Run `./tf-wrapper.sh plan non-production` and review output.
+1. Run `./tf-wrapper.sh plan non-production` and review output.
+1. Run `./tf-wrapper.sh validate non-production $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply non-production`
 1. Run `./tf-wrapper.sh init development`
 1. Run `./tf-wrapper.sh plan development` and review output.
+1. Run `./tf-wrapper.sh validate development $(pwd)/../policy-library <YOUR_CLOUD_BUILD_PROJECT_ID>` and check for violations.
 1. Run `./tf-wrapper.sh apply development`
 
 If you received any errors or made any changes to the Terraform config or `terraform.tfvars` you must re-run `./tf-wrapper.sh plan <env>` before run `./tf-wrapper.sh apply <env>`