Local kibana with ECH hosted data and monitoring clusters

main.tf

provider "ec" {}
provider "elasticstack" {}

# Elasticsearch backend for local kibana instance
# no ECH-hosted kibana or integration server
resource "ec_deployment" "headless" {
  name                   = "headless"

  region                 = "gcp-${var.region}"
  version                = "${var.elastic_version}"
  deployment_template_id = "gcp-cpu-optimized"

  tags                   = {
    division = "engineering"
    org      = "obs"
    team     = "ingest"
  }

  elasticsearch = {
    hot = {
      instance_configuration_id = "gcp.es.datahot.n2.68x10x45"
      size = "8g"
      autoscaling = {}
      zone_count = 1
    }

    remote_cluster = [
      {
        deployment_id = ec_deployment.monitoring.id
        alias = ec_deployment.monitoring.name
        ref_id = ec_deployment.monitoring.elasticsearch.ref_id
      }
    ]
  }
}

# APM ingestion target for local kibana instance
# ECH-hosted kibana and integration server to provide APM Server
resource "ec_deployment" "monitoring" {
  name                   = "monitoring"

  region                 = "gcp-${var.region}"
  version                = "${var.elastic_version}"
  deployment_template_id = "gcp-cpu-optimized"

  tags                   = {
    division = "engineering"
    org      = "obs"
    team     = "ingest"
  }

  elasticsearch = {
    hot = {
      instance_configuration_id = "gcp.es.datahot.n2.68x10x45"
      size = "8g"
      autoscaling = {}
      zone_count = 1
    }
  }

  kibana = {
    zone_count = 1
  }

  integrations_server = {
    zone_count = 1
  }
}

output "headless_deployment_id" {
  value = ec_deployment.headless.id
}

output "monitoring_deployment_id" {
  value = ec_deployment.headless.id
}


output "headless_elasticsearch_url" {
  value       = ec_deployment.headless.elasticsearch.https_endpoint
  description = "Elasticsearch endpoint for the Central Cluster"
}

output "headless_username" {
  value       = ec_deployment.headless.elasticsearch_username
  description = "Username for the Central Cluster"
}

output "headless_password" {
  value       = ec_deployment.headless.elasticsearch_password
  description = "Password for the Central Cluster"
  sensitive   = true
}

output "apm_secret_token" {
  value = ec_deployment.monitoring.apm_secret_token
  sensitive = true
}

output "apm_server_url" {
  value = ec_deployment.monitoring.integrations_server.https_endpoint
}

output "monitoring_cluster_kibana_url" {
  value       = ec_deployment.monitoring.kibana.https_endpoint
  description = "Kibana URL for the Remote Cluster"
}

output "monitoring_cluster_elasticsearch_url" {
  value       = ec_deployment.monitoring.elasticsearch.https_endpoint
  description = "Elasticsearch endpoint for the Remote Cluster"
}

output "monitoring_cluster_username" {
  value       = ec_deployment.monitoring.elasticsearch_username
  description = "Username for the Remote Cluster"
}

output "monitoring_cluster_password" {
  value       = ec_deployment.monitoring.elasticsearch_password
  description = "Password for the Remote Cluster"
  sensitive   = true
}

locals {
  kibana_yml_template = <<-EOF
    # create with:
    # curl -X POST -H "Content-Type: application/json" \
    #   -u "$${username}:$${password}" \
    #   "$${elasticsearch_url}/_security/service/elastic/kibana/credential/token/kibana_service_token"
    #
    elasticsearch.serviceAccountToken: ""

    elasticsearch.hosts: ["$${elasticsearch_url}"]
    elastic.apm.active: true
    elastic.apm.serverUrl: "$${apm_server_url}"
    elastic.apm.secretToken: "$${apm_server_token}"
  EOF
}

output "local_kibana_yml" {
  value = templatestring(local.kibana_yml_template, {
    elasticsearch_url = ec_deployment.headless.elasticsearch.https_endpoint
    apm_server_token  = ec_deployment.monitoring.apm_secret_token
    apm_server_url    = ec_deployment.monitoring.integrations_server.https_endpoint
    username          = ec_deployment.headless.elasticsearch_username
    password          = ec_deployment.headless.elasticsearch_password
  })

  sensitive = true
}

vars.tf

variable "region" {
  description = "GCP region"
  default = "us-east4"
}

variable "elastic_version" {
  description = "Elastic stack version"
  default = "8.18.0"
}

versions.tf

terraform {
  required_version = ">= 1.0.0"

  required_providers {
    ec = {
      source  = "elastic/ec"
    }
    elasticstack = {
      source  = "elastic/elasticstack"
    }
  }
}