This installs all the components needed to collect metrics and traces with OTel, store them in Prometheus/Jaeger, and view them in Grafana.

setup-otel.sh

#!/usr/bin/env bash

header=$'\e[1;33m'
reset=$'\e[0m'

function header_text {
	echo "$header$*$reset"
}

header_text "Installing Cert Manager"

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml

header_text "Waiting for Cert Manager to become ready"

kubectl wait deployment --all --timeout=-1s --for=condition=Available -n cert-manager

header_text "Installing OTel Operator"

kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml

header_text "Waiting for OTel Operator to become ready"

kubectl wait deployment --all --timeout=-1s --for=condition=Available -n opentelemetry-operator-system

header_text "Installing Grafana Operator"

kubectl create namespace grafana
helm upgrade -i grafana-operator oci://ghcr.io/grafana/helm-charts/grafana-operator --version v5.18.0 --namespace grafana

header_text "Waiting for Grafana operator to become ready"

kubectl wait deployment --all --timeout=-1s --for=condition=Available -n grafana

header_text "Installing Prometheus Operator"

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
kubectl create namespace prometheus
helm install prometheus prometheus-community/kube-prometheus-stack --namespace prometheus --set grafana.enabled=false

header_text "Waiting for Prometheus operator to become ready"

kubectl wait deployment --all --timeout=-1s --for=condition=Available --namespace prometheus

header_text "Creating Jaeger Instance"

kubectl create namespace jaeger
kubectl apply -f - <<EOF
apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
  name: jaeger-inmemory
  namespace: jaeger
spec:
  image: jaegertracing/jaeger:latest
  ports:
  - name: jaeger
    port: 16686
  config:
    service:
      extensions: [jaeger_storage, jaeger_query]
      pipelines:
        traces:
          receivers: [otlp]
          exporters: [jaeger_storage_exporter]
    extensions:
      jaeger_query:
        storage:
          traces: memstore
      jaeger_storage:
        backends:
          memstore:
            memory:
              max_traces: 100000
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: "0.0.0.0:4317"
          http:
            endpoint: "0.0.0.0:4318"
    exporters:
      jaeger_storage_exporter:
        trace_storage: memstore
EOF

header_text "Creating Otel Collector"

kubectl create namespace knative-observability

kubectl apply -f - <<EOF
apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
  name: knative
  namespace: knative-observability
spec:
  config:
    receivers:
      opencensus:
        endpoint: "0.0.0.0:55678"
      otlp:
        protocols:
          grpc:
            endpoint: "0.0.0.0:4317"
          http:
            endpoint: "0.0.0.0:4318"
    processors:
      batch:
        timeout: 5s
    exporters:
      debug:
        verbosity: detailed
      prometheus:
        endpoint: "0.0.0.0:8889"
        namespace: default
      otlp:
        endpoint: "jaeger-inmemory-collector.jaeger.svc.cluster.local:4317"
        tls:
          insecure: true
    service:
      pipelines:
        traces:
          receivers: [opencensus, otlp]
          processors: [batch]
          exporters: [debug, otlp]
        metrics:
          receivers: [opencensus, otlp]
          processors: [batch]
          exporters: [debug, prometheus]
EOF

header_text "Creating Prometheus ServiceMonitor"

kubectl apply -f - <<EOF
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: otel-collector-metrics
  namespace: knative-observability
  labels:
    release: prometheus
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: knative-collector
  endpoints:
  - port: prometheus
    path: /metrics
EOF

header_text "Creating Grafana instance"

kubectl apply -f - <<EOF
apiVersion: grafana.integreatly.org/v1beta1
kind: Grafana
metadata:
  name: grafana
  namespace: knative-observability
  labels:
    dashboards: grafana
spec:
  config:
    security:
      admin_user: root
      admin_password: secret
EOF

kubectl apply -f - <<EOF
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaDatasource
metadata:
  name: knative-metrics
  namespace: knative-observability
spec:
  instanceSelector:
    matchLabels:
      dashboards: "grafana"
  datasource:
    name: prometheus
    type: prometheus
    access: proxy
    url: http://prometheus-kube-prometheus-prometheus.prometheus.svc.cluster.local:9090
    isDefault: true
    jsonData:
      'tlsSkipVerify': true
      'timeInterval': "5s"
EOF

kubectl apply -f - <<EOF
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaDatasource
metadata:
  name: knative-traces
  namespace: knative-observability
spec:
  instanceSelector:
    matchLabels:
      dashboards: "grafana"
  datasource:
    name: jaeger
    type: jaeger
    access: proxy
    url: http://jaeger-inmemory-collector.jaeger.svc.cluster.local:16686
    jsonData:
      'tlsSkipVerify': true
      'timeInterval': "5s"
EOF

To access the grafana dashboard, run:

kubectl port-forward -n knative-observability svc/grafana-service 3000:3000

The user & pass are root and secret