From b94527dd24752a59257dc4b9d9f766604dc54595 Mon Sep 17 00:00:00 2001 From: Chris ter Beke <1134120+ChrisTerBeke@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:50:49 +0100 Subject: [PATCH] Remove tinyproxy, use SSH directly --- README.md | 31 +++++++--- bastion.tf | 20 +++---- kubernetes.tf | 4 +- templates/bastion-cloud-init.yaml | 5 ++ templates/gke-bastion-startup.tftpl.sh | 7 --- vpc.tf | 82 ++++++++++++-------------- 6 files changed, 76 insertions(+), 73 deletions(-) create mode 100644 templates/bastion-cloud-init.yaml delete mode 100644 templates/gke-bastion-startup.tftpl.sh diff --git a/README.md b/README.md index 42614b7..c2e33a3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,38 @@ # Proxying your way in GKE -Securely connect to a Google Kubernetes Engine (GKE) Cluster using Terraform, Tinyproxy, and Identity-Aware Proxy. +Securely connect to a Google Kubernetes Engine (GKE) Cluster using Terraform, SSH, and Identity-Aware Proxy. ## Features This configuration provides ready-to-use resources for production: + - VPC with Private Google Access enabled. - Google Kubernetes Engine (GKE) Cluster. -- Managed Instance Group (MIG) hosting a single instance running Tinyproxy. +- Managed Instance Group (MIG) hosting a single instance running SSH. - Preconfigured Kubernetes Terraform Provider. -## Setting Up a Secure Tunnel Using IAP and Tinyproxy +## Setting Up a Secure Tunnel Using IAP and SSH + +To create a secure tunnel using Identity-Aware Proxy (IAP) and SSH: + +```bash +CLOUDSDK_PYTHON_SITEPACKAGES=1 gcloud compute ssh \ + --project= \ + --zone= \ + --tunnel-through-iap \ + --ssh-flag="-N -f -D 8888" \ +``` + +To kill the tunnel: + +```bash +kill -9 $(shell lsof 8888 > /dev/null 2> /dev/null || : +``` -To create a secure tunnel using Identity-Aware Proxy (IAP) and Tinyproxy: +To test the connection: ```bash -CLOUDSDK_PYTHON_SITEPACKAGES=1 gcloud compute ssh --tunnel-through-iap --project= --zone= --ssh-flag='-4 -L8888:localhost:8888 -N -q -f' +HTTPS_PROXY=socks5://127.0.0.1:8888 kubectl cluster-info ``` When using GitHub Actions: @@ -31,10 +48,10 @@ When using GitHub Actions: - name: Set up gcloud uses: google-github-actions/setup-gcloud@v2 -- name: Create a secure tunnel using IAP and Tinyproxy +- name: Create a secure tunnel using IAP and SSH run: | gcloud components install gke-gcloud-auth-plugin --quiet - gcloud compute ssh ${{ inputs.gcp_bastion_host }} --tunnel-through-iap --project=${{ inputs.gcp_bastion_project }} --zone=${{ inputs.gcp_bastion_zone }} --ssh-flag="-4 -L8888:localhost:8888 -N -q -f" + gcloud compute ssh ${{ inputs.gcp_bastion_host }} --tunnel-through-iap --project=${{ inputs.gcp_bastion_project }} --zone=${{ inputs.gcp_bastion_zone }} --ssh-flag="-N -f -D 8888" - name: Set up Terraform uses: hashicorp/setup-terraform@v3 diff --git a/bastion.tf b/bastion.tf index deb538c..e25111f 100644 --- a/bastion.tf +++ b/bastion.tf @@ -16,10 +16,9 @@ resource "google_service_account" "gke_bastion" { project = "your-project" } -# dynamically fetch the latest Debian 11 image -data "google_compute_image" "debian" { - family = "debian-11" - project = "debian-cloud" +data "google_compute_image" "cos" { + family = "cos-105-lts" + project = "cos-cloud" } resource "google_compute_instance_template" "gke_bastion" { @@ -28,6 +27,7 @@ resource "google_compute_instance_template" "gke_bastion" { instance_description = "GKE bastion instance" region = "us-central1" machine_type = "e2-micro" + project = "your-project" tags = [ "gke-bastion", @@ -37,7 +37,7 @@ resource "google_compute_instance_template" "gke_bastion" { google-logging-enabled = true enable-oslogin = true block-project-ssh-keys = true - startup-script = templatefile("${path.module}/templates/gke-bastion-startup.tftpl.sh", {}) + user-data = file("${path.module}/templates/bastion-cloud-init.yaml") } scheduling { @@ -47,7 +47,7 @@ resource "google_compute_instance_template" "gke_bastion" { # ephemeral OS boot disk disk { - source_image = data.google_compute_image.debian.self_link + source_image = data.google_compute_image.cos.self_link auto_delete = true boot = true disk_type = "pd-ssd" @@ -70,8 +70,6 @@ resource "google_compute_instance_template" "gke_bastion" { enable_integrity_monitoring = true } - project = "your-project" - # instance Templates cannot be updated after creation. # in order to update an Instance Template, Terraform will destroy the existing resource and create a replacement lifecycle { @@ -84,13 +82,13 @@ resource "google_compute_instance_group_manager" "gke_bastion" { base_instance_name = "gke-bastion" zone = "us-central1-a" description = "Manages the lifecycle of the GKE bastion instance" + project = "your-project" + target_size = 1 version { instance_template = google_compute_instance_template.gke_bastion.id } - target_size = 1 - # because we allocated only 1 static internal IP, we can't use rolling updates. update_policy { type = "PROACTIVE" @@ -100,6 +98,4 @@ resource "google_compute_instance_group_manager" "gke_bastion" { max_unavailable_fixed = 1 replacement_method = "RECREATE" } - - project = "your-project" } diff --git a/kubernetes.tf b/kubernetes.tf index d1017af..b9fecba 100644 --- a/kubernetes.tf +++ b/kubernetes.tf @@ -1,7 +1,7 @@ provider "kubernetes" { - host = "https://${google_container_cluster.example.endpoint}" + host = "https://${google_container_cluster.example.endpoint}" # the `proxy_url` references the locally secured tunnel created by Identity-Aware Proxy. - proxy_url = "http://localhost:8888" + proxy_url = "socks5://127.0.0.1:8888" cluster_ca_certificate = base64decode(google_container_cluster.example.master_auth.0.cluster_ca_certificate) # we use this for authenticating natively with GKE, rather than relying on tokens and certificates. diff --git a/templates/bastion-cloud-init.yaml b/templates/bastion-cloud-init.yaml new file mode 100644 index 0000000..8131bb2 --- /dev/null +++ b/templates/bastion-cloud-init.yaml @@ -0,0 +1,5 @@ +#cloud-config + +runcmd: + - [ sed, -i, 's/PermitTunnel no/PermitTunnel yes/g', /etc/ssh/sshd_config ] + - [ systemctl, restart, sshd ] diff --git a/templates/gke-bastion-startup.tftpl.sh b/templates/gke-bastion-startup.tftpl.sh deleted file mode 100644 index 3c66583..0000000 --- a/templates/gke-bastion-startup.tftpl.sh +++ /dev/null @@ -1,7 +0,0 @@ -sudo apt-get update -y -sudo apt-get install -y tinyproxy -sudo mkdir -p /etc/systemd/system/tinyproxy.service.d/ -echo -e '[Service]\nRestart=always' | sudo tee /etc/systemd/system/tinyproxy.service.d/override.conf -echo -e 'Allow localhost' | sudo tee -a /etc/tinyproxy/tinyproxy.conf -sudo systemctl daemon-reload -sudo systemctl restart tinyproxy diff --git a/vpc.tf b/vpc.tf index 5c5c30c..e903b34 100644 --- a/vpc.tf +++ b/vpc.tf @@ -39,11 +39,12 @@ resource "google_compute_subnetwork" "bastion" { region = "us-central1" network = google_compute_network.default.id private_ip_google_access = true + project = "your-project" + log_config { aggregation_interval = "INTERVAL_5_SEC" flow_sampling = 0.5 } - project = "your-project" } # Create a subnet for the GKE cluster @@ -53,19 +54,22 @@ resource "google_compute_subnetwork" "default" { region = "us-central1" network = google_compute_network.default.id private_ip_google_access = true + project = "your-project" + secondary_ip_range { range_name = "gke-services" ip_cidr_range = "10.1.0.0/16" } + secondary_ip_range { range_name = "gke-pods" ip_cidr_range = "10.2.0.0/20" } + log_config { aggregation_interval = "INTERVAL_5_SEC" flow_sampling = 0.5 } - project = "your-project" } # A route for public internet traffic @@ -81,10 +85,11 @@ resource "google_compute_route" "public_internet" { # Allow internal traffic within the network resource "google_compute_firewall" "allow_internal_ingress" { - name = "allow-internal-ingress" - network = google_compute_network.default.name - - direction = "INGRESS" + name = "allow-internal-ingress" + network = google_compute_network.default.name + direction = "INGRESS" + source_ranges = ["10.128.0.0/9"] + project = "your-project" allow { protocol = "icmp" @@ -99,45 +104,32 @@ resource "google_compute_firewall" "allow_internal_ingress" { protocol = "udp" ports = ["0-65535"] } - - source_ranges = ["10.128.0.0/9"] - - project = "your-project" } # Allow incoming TCP traffic from Identity-Aware Proxy (IAP) resource "google_compute_firewall" "allow_iap_tcp_ingress" { - name = "allow-iap-tcp-ingress" - network = google_compute_network.default.name - - direction = "INGRESS" + name = "allow-iap-tcp-ingress" + network = google_compute_network.default.name + direction = "INGRESS" + project = "your-project" + source_ranges = [local.iap_tcp_forwarding_cidr_range] allow { protocol = "tcp" - ports = ["22"] } - - source_ranges = [ - local.iap_tcp_forwarding_cidr_range, - ] - - project = "your-project" } # By default, deny all egress traffic resource "google_compute_firewall" "deny_all_egress" { - name = "deny-all-egress" - network = google_compute_network.default.name - - direction = "EGRESS" + name = "deny-all-egress" + network = google_compute_network.default.name + project = "your-project" + direction = "EGRESS" + destination_ranges = ["0.0.0.0/0"] deny { protocol = "all" } - - destination_ranges = ["0.0.0.0/0"] - - project = "your-project" } resource "google_compute_network_firewall_policy" "gke_bastion" { @@ -148,26 +140,25 @@ resource "google_compute_network_firewall_policy" "gke_bastion" { # Allow GKE bastion instances to communicate with only the FQDNs to install packages resource "google_compute_network_firewall_policy_rule" "allow_gke_bastion_egress_to_packages_debian_org" { - firewall_policy = google_compute_network_firewall_policy.gke_bastion.name - priority = 1000 - + firewall_policy = google_compute_network_firewall_policy.gke_bastion.name + priority = 1000 action = "allow" direction = "EGRESS" target_service_accounts = [google_service_account.gke_bastion.email] + project = "your-project" match { - layer4_configs { - ip_protocol = "tcp" - } - dest_fqdns = [ "packages.debian.org", "debian.map.fastly.net", "deb.debian.org", "packages.cloud.google", ] + + layer4_configs { + ip_protocol = "tcp" + } } - project = "your-project" } # Allow private google access egress traffic @@ -178,6 +169,7 @@ resource "google_compute_firewall" "allow_private_google_access_egress" { priority = 4000 direction = "EGRESS" target_tags = [] + project = "your-project" destination_ranges = [ local.private_google_access_cidr_range, @@ -192,18 +184,17 @@ resource "google_compute_firewall" "allow_private_google_access_egress" { log_config { metadata = "INCLUDE_ALL_METADATA" } - project = "your-project" } resource "google_compute_router" "default" { name = "router" region = "us-central1" network = google_compute_network.default.id + project = "your-project" bgp { asn = 64514 } - project = "your-project" } # For redundancy, create two NAT IPs @@ -222,12 +213,12 @@ resource "google_compute_router_nat" "default" { nat_ip_allocate_option = "MANUAL_ONLY" nat_ips = google_compute_address.nat.*.self_link source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" + project = "your-project" log_config { enable = true filter = "ERRORS_ONLY" } - project = "your-project" } # Create private DNS zones to route traffic to private google access IPs @@ -236,6 +227,7 @@ resource "google_dns_managed_zone" "private_service_access" { name = each.key dns_name = each.value.dns visibility = "private" + project = "your-project" private_visibility_config { dynamic "networks" { @@ -246,7 +238,6 @@ resource "google_dns_managed_zone" "private_service_access" { } } } - project = "your-project" } resource "google_dns_record_set" "a_records" { @@ -295,15 +286,16 @@ resource "google_compute_route" "restricted_google_access" { # A network management connectivity test to verify the GKE bastion instance can communicate with the GKE cluster resource "google_network_management_connectivity_test" "gke_bastion_to_gke" { - name = "gke-bastion-to-gke" + name = "gke-bastion-to-gke" + protocol = "TCP" + project = "your-project" + source { ip_address = google_compute_address.gke_bastion.address } + destination { port = 443 ip_address = google_container_cluster.example.endpoint } - - protocol = "TCP" - project = "your-project" }