A Better Nomad w/ Consul Connect Service Mesh Job Example

service_mesh_job_example.hcl

# Service Mesh Job Example
# Adapted from: https://developer.hashicorp.com/nomad/docs/job-specification/connect
#
# Contains many more features, such as dynamic ports, podman, canary deployment, and API service hiding
#
# Tested w/ Nomad 1.11.1 & Consul 1.22.2 
# Using CNI Plugins 1.9.0: https://github.com/containernetworking/plugins/releases/download/v1.9.0/cni-plugins-linux-amd64-v1.9.0.tgz
#
# It's probably a good idea to declare a connect -> meta in /etc/nomad.d/nomad.hcl for the envoy image defaults:
#
#  # Meta data available to jobs
#  # https://developer.hashicorp.com/nomad/docs/configuration/client#meta
#  client {
#    # ...
#    meta {
#      # Add helper variable for determining the physical host in Nomad Jobs, if needed
#      physical_host = "{{ fully_qualified_domain_name }}"
#
#      # Setup default envoy images based on Nomad's built-in envoy version
#      # Nomad will use heuristic to run podman for sidecars: https://github.com/hashicorp/nomad/pull/17065
#      #
#      # Replace "docker.io/envoyproxy" with own CNCF Distribution Registry if required for caching/security
#      "connect.gateway_image" = "docker.io/envoyproxy/envoy:v1.35.8"
#      "connect.sidecar_image" = "docker.io/envoyproxy/envoy:v1.35.8"
#    }
#    # ...
#  }
#

# Job definition
# https://developer.hashicorp.com/nomad/docs/job-specification/job
job "countdash" {
  # Data centres this job can be run
  # https://developer.hashicorp.com/nomad/docs/job-specification/job#datacenters
  datacenters = ["dc1"]

  # Strategy for failed allocations and migrations
  # https://developer.hashicorp.com/nomad/docs/job-specification/reschedule
  reschedule {
    delay          = "10s"
    delay_function = "constant"
    unlimited      = true
  }

  # Strategy for transitioning from one job plan to the next (eg Rolling Updates & Canaries)
  # https://developer.hashicorp.com/nomad/docs/job-specification/update
  update {
    canary            = 1
    auto_promote      = true
    auto_revert       = true
    max_parallel      = 2
  }

  # Strategy for moving jobs when draining nodes
  # https://developer.hashicorp.com/nomad/docs/job-specification/migrate
  migrate {
    max_parallel     = 1
    min_healthy_time = "10s"
    healthy_deadline = "5m"
  }

  # Web API Group (Backend example)
  group "api" {
    # Network requirements
    # https://developer.hashicorp.com/nomad/docs/job-specification/network
    network {
      mode = "bridge"

      # Don't publicly expose API, only available through service mesh
      # https://developer.hashicorp.com/nomad/docs/job-specification/expose
      port "api-port" {
        to = -1
      }
    }

    # Service Registration
    # https://developer.hashicorp.com/nomad/docs/job-specification/service
    service {
      # Advertised name in Consul
      # https://developer.hashicorp.com/nomad/docs/job-specification/service#name
      name = "count-api"

      # Advertised port in Consul  (see network stanza above)
      # https://developer.hashicorp.com/nomad/docs/job-specification/service#port
      port = "api-port"

      # Consul Service Mesh Sidecar (Producer)
      # https://developer.hashicorp.com/nomad/docs/job-specification/connect#using-sidecar-service
      connect {
        sidecar_service {
          proxy {
            # Port to map for sidecar service: sidecar -> port in container
            local_service_port = 9001

            # Exposes health check for public use, even though the rest of the API is not available
            # https://developer.hashicorp.com/nomad/docs/job-specification/expose
            expose {
              path {
                path            = "/health"
                protocol        = "http"
                local_path_port = 9001
                listener_port   = "api-port"
              }
            }
           }
        }
      }

      # Health Check
      # https://developer.hashicorp.com/nomad/docs/job-specification/check
      check {
        # Expose cannot be done here with dynamic ports and side cars.  See connect -> sidecar_service -> proxy -> expose above
        # https://developer.hashicorp.com/nomad/docs/job-specification/check#expose
        # expose  = "true"
        type     = "http"
        port     = "api-port"
        name     = "api-health"
        path     = "/health"
        interval = "10s"
        timeout  = "3s"
      }
    }

    # Task definition
    # https://developer.hashicorp.com/nomad/docs/job-specification/task
    task "web" {
      driver = "podman"

      config {
        image = "docker.io/hashicorpdev/counter-api:v3"  # Image contains HTTP service on localhost:9001
        ports = ["api-port"]
      }
    }
  }

  # Dashboard WebUI Group (Frontend example)
  group "dashboard" {
    # Network requirements
    # https://developer.hashicorp.com/nomad/docs/job-specification/network
    network {
      mode = "bridge"

      # Consul-exposed dynamic port
      # https://developer.hashicorp.com/nomad/docs/job-specification/network#port-parameters
      port "dashboard-port" {
        to = 9002
      }
    }

    # Service Registration
    # https://developer.hashicorp.com/nomad/docs/job-specification/service
    service {
      # Advertised name in Consul
      # https://developer.hashicorp.com/nomad/docs/job-specification/service#name
      name = "count-dashboard"

      # Advertised port in Consul (see network stanza above)
      # https://developer.hashicorp.com/nomad/docs/job-specification/service#port
      port = "dashboard-port"

      # Consul Service Mesh Sidecar (Consumer)
      # https://developer.hashicorp.com/nomad/docs/job-specification/connect#using-sidecar-service
      connect {
        sidecar_service {
          proxy {
            upstreams {
              # Creates HCL variables for this Job
              #
              # * NOMAD_UPSTREAM_ADDR_{{ destination_name | replace ('-', '_') }} = localhost:8080
              # * NOMAD_UPSTREAM_IP_{{ destination_name | replace ('-', '_') }} = localhost
              # * NOMAD_UPSTREAM_PORT_{{ destination_name | replace ('-', '_') }} = 8080
              #
              destination_name = "count-api"
              local_bind_port  = 8080
            }
          }
        }
      }
    }

    # Task definition
    # https://developer.hashicorp.com/nomad/docs/job-specification/task
    task "dashboard" {
      driver = "podman"

      # Image specific environment variables, for mapping dynamic ports into container
      # Image = counter-dashboard:v3
      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "docker.io/hashicorpdev/counter-dashboard:v3" # Image contains HTTP service on localhost:9002
        ports = ["dashboard-port"]
      }
    }
  }
}