GKE terraform - 2020.10.06

gke_main.tf

locals {
  cluster_name = "${var.name}-k8s-cluster"

  gke_sa_iam_roles = [
    "roles/logging.logWriter",
    "roles/monitoring.metricWriter",
    "roles/monitoring.viewer",
    "roles/stackdriver.resourceMetadata.writer",
    "roles/storage.objectViewer",
  ]
}

variable "module_depends_on" {
  # the value doesn't matter; we're just using this variable
  # to propagate dependencies.
  type    = any
  default = []
}

############################################################### Service Accounts

resource "google_service_account" "gke-node-sa" {
  account_id   = "${local.cluster_name}-node-sa"
  display_name = "Service Account for ${local.cluster_name} GKE nodes"
  project      = var.project_id
}

resource "google_project_iam_member" "gke-node-sa-iam" {
  count   = length(local.gke_sa_iam_roles)
  project = var.project_id
  role    = element(local.gke_sa_iam_roles, count.index)
  member  = "serviceAccount:${google_service_account.gke-node-sa.email}"
}

############################################################################ NAT

#------------------#
# Create Cloud NAT #
#------------------#

// Create an external NAT IP
resource "google_compute_address" "nat" {
  name    = "${local.cluster_name}-nat-ip"
  project = var.project_id
  region  = var.region
}

// Create a cloud router for use by the Cloud NAT
resource "google_compute_router" "router" {
  name    = "${local.cluster_name}-cloud-router"
  project = var.project_id
  region  = var.region

  network = var.network_name

  bgp {
    asn = 64514
  }
}

// Create a NAT router so the nodes can reach DockerHub, etc
module "cloud-nat" {
  source  = "terraform-google-modules/cloud-nat/google"
  version = "~> 1.3.0"

  project_id = var.project_id
  name       = "${local.cluster_name}-nat"

  region        = var.region
  router        = google_compute_router.router.name
}

#################################################################### GKE CLUSTER

module "private_gke_cluster" {
  source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster"
  version = "~> 11.1.0"

  project_id = var.project_id
  name       = local.cluster_name

  authenticator_security_group = var.authenticator_security_group

  regional               = true
  region                 = var.region
  network                = var.network_name
  subnetwork             = var.subnetwork_name
  ip_range_pods          = var.ip_range_pods
  ip_range_services      = var.ip_range_services

  http_load_balancing        = true
  horizontal_pod_autoscaling = true
  network_policy             = false
  istio                      = true

  enable_private_endpoint = false
  enable_private_nodes    = true
  master_ipv4_cidr_block = var.master_node_range

  create_service_account = false
  service_account = google_service_account.gke-node-sa.email

  remove_default_node_pool = false // true

  node_pools = [
    {
      name               = "default-node-pool"
      machine_type       = "e2-medium"
      min_count          = 2
      max_count          = 3
      local_ssd_count    = 0
      disk_size_gb       = 10
      disk_type          = "pd-standard"
      image_type         = "COS"
      auto_repair        = true
      auto_upgrade       = true
      service_account    = google_service_account.gke-node-sa.email
      preemptible        = false
      initial_node_count = 1
    },
  ]

  node_pools_oauth_scopes = {
    all = []

    default-node-pool = [
      "https://www.googleapis.com/auth/cloud-platform",
    ]
  }

  node_pools_labels = {
    all = {}

    default-node-pool = {}
  }

  node_pools_metadata = {
    all = {}

    default-node-pool = {
      # node-pool-metadata-custom-value = "my-node-pool"
    }
  }

  node_pools_taints = {
    all = []

    default-node-pool = [
      # {
      #   key    = "default-node-pool"
      #   value  = true
      #   effect = "PREFER_NO_SCHEDULE"
      # },
    ]
  }

  node_pools_tags = {
    all = []

    default-node-pool = [
      # "default-node-pool",
    ]
  }

}

gke_variables.tf

variable "project_id" {
  type = string

  description = "The project ID to host the cluster in"
}

variable "region" {
  type = string

  description = "The region to host the cluster in"
}

variable "name" {
  type = string

  description = "The name (prefix) where the cluster name will be derived from"
}

variable "authenticator_security_group" {
  type = string

  description = <<EOF
authenticator_security_group	The name of the RBAC security group for use with
Google security groups in Kubernetes RBAC. Group name must be in format
[email protected]"
EOF
}

variable "network_name" {
  type = string

  description = "The name of the VPC network where the cluster is in"
}

variable "subnetwork_name" {
  type = string

  description = "The name of the VPC subnetwork where the cluster is in"
}

variable "ip_range_pods" {
  type = string

  description = "The IP range for the pods"
}

variable "ip_range_services" {
  type = string

  description = "The IP range for the services"
}

variable "master_node_range" {
  type = string

  description = "The IP range in CIDR notation to use for the hosted master network"
}

main.tf

########################################################################### Data

data "terraform_remote_state" "projects" {
  backend   = "gcs"
  workspace = terraform.workspace

  config = {
    bucket = "io-treasuroo-tf"
    prefix = "projects"
  }
}

locals {
  gcp_project_id     = data.terraform_remote_state.projects.outputs.project_id
  gcp_location_parts = split("-", var.gcp_location)
  gcp_region         = format("%s-%s", local.gcp_location_parts[0], local.gcp_location_parts[1])
}

###################################################################### Providers

provider "google" {
  version = "~> 3.40"

  project = local.gcp_project_id
  region  = local.gcp_region
}

provider "google-beta" {
  version = "~> 3.5"

  project = local.gcp_project_id
  region  = local.gcp_region
}

######################################################################## Modules

module "network" {
  source = "./networking/network"

  project_id = local.gcp_project_id
  region     = local.gcp_region
  name       = terraform.workspace
  google_apis_route_name = var.google_apis_route_name
}

module "gke" {
  source = "./gke"

  project_id           = local.gcp_project_id
  region               = local.gcp_region
  name                 = terraform.workspace
  network_name         = module.network.network_name
  subnetwork_name      = module.network.subnet_name
  ip_range_pods        = module.network.ip_range_pods
  ip_range_services    = module.network.ip_range_services
  master_node_range    = "172.16.0.0/28"
  authenticator_security_group = "[email protected]"

  module_depends_on = [
    module.ip
  ]
}

networking_network_main.tf

locals {
  region         = var.region
  network_name   = "${var.name}-vpc"
  subnet_name    = "${var.name}-subnet"
  pods_range     = "${local.subnet_name}-pods"
  services_range = "${local.subnet_name}-services"
}

module "no_internet_network" {
    source  = "terraform-google-modules/network/google"
    version = "~> 2.5"

    project_id   = var.project_id
    network_name = local.network_name
    routing_mode = "GLOBAL"
    delete_default_internet_gateway_routes = true

    subnets = [
        {
            subnet_name           = local.subnet_name
            subnet_ip             = "10.0.0.0/24"
            subnet_region         = var.region
            subnet_private_access = true
            subnet_flow_logs      = true
        },
    ]

    secondary_ranges = {
        "${local.subnet_name}" = [
            {
                range_name    = local.pods_range
                ip_cidr_range = "10.10.0.0/16"
            },
            {
                range_name    = local.services_range
                ip_cidr_range = "10.20.0.0/24"
            },
        ]
    }

    # routes = [
    #     {
    #         name              = "${var.google_apis_route_name}"
    #         destination_range = "199.36.153.4/30"
    #         next_hop_internet = "true"
    #     },
    # ]

}

networking_network_output.tf

output "network_name" {
  value       = module.no_internet_network.network_name
  description = "The unique name of the network"
}

output "network" {
  value = module.no_internet_network.network
  description = "The created network resource"
}

output "subnets" {
  value = module.no_internet_network.subnets
  description = "The created subnetwork resources"
}

output "subnet_name" {
  value = local.subnet_name
  description = "The created subnetwork resource names"
}

output "ip_range_pods" {
  value = local.pods_range
  description = "The name of the secondary range for pods"
}

output "ip_range_services" {
  value = local.services_range
  description = "The name of the secondary range for services"
}

networking_network_variables.tf

variable "project_id" {
  type = string

  description = "The project ID to host the network in"
}

variable "region" {
  type = string

  description = "The region to host the cluster in"
}

variable "name" {
  type = string

  description = "The name (prefix) where vpc resource names will be derived from"
}

variable "google_apis_route_name" {
  type = string

  default = "google-apis"

  description = "Name for the route to restricted Google APIs"
}