Skip to content

Instantly share code, notes, and snippets.

@jpkrohling

jpkrohling/cr-load-balancer.yaml

Last active May 8, 2024 11:21
Download ZIP
Load-balancer reproducer
Raw
cr-load-balancer.yaml
apiVersion: v1
kind: Namespace
metadata:
name: observability
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: observability
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: prometheus
namespace: observability
rules:
- apiGroups: [""]
resources:
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: prometheus
namespace: observability
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: observability
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: loadbalancer-role
namespace: observability
rules:
- apiGroups:
- ""
resources:
- endpoints
verbs: ["get", "list", "watch"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: loadbalancer
namespace: observability
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: loadbalancer-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: loadbalancer-role
subjects:
- kind: ServiceAccount
name: loadbalancer
namespace: observability
---
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
name: loadbalancer
namespace: observability
spec:
image: otel/opentelemetry-collector-contrib:0.100.0
serviceAccount: loadbalancer
config: |
receivers:
otlp:
protocols:
grpc:
processors:
exporters:
loadbalancing:
protocol:
otlp:
tls:
insecure: true
resolver:
k8s:
service: backends-collector-headless.observability
service:
pipelines:
traces:
receivers:
- otlp
processors: []
exporters:
- loadbalancing
---
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
name: backends
namespace: observability
spec:
replicas: 10
config: |
receivers:
otlp:
protocols:
grpc:
processors:
exporters:
logging:
service:
pipelines:
traces:
receivers:
- otlp
processors: []
exporters:
- logging
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: backends-monitor
namespace: observability
labels:
scrape: "yes"
spec:
selector:
matchLabels:
app.kubernetes.io/name: backends-collector-monitoring
endpoints:
- port: monitoring
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: loadbalancer-monitor
namespace: observability
labels:
scrape: "yes"
spec:
selector:
matchLabels:
app.kubernetes.io/name: loadbalancer-collector-monitoring
endpoints:
- port: monitoring
---
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: prometheus
namespace: observability
spec:
serviceAccountName: prometheus
serviceMonitorSelector:
matchLabels:
scrape: "yes"
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: grafana
name: grafana
namespace: observability
spec:
selector:
matchLabels:
app: grafana
template:
metadata:
labels:
app: grafana
spec:
containers:
- name: grafana
image: grafana/grafana:latest
ports:
- containerPort: 3000
name: http-grafana
protocol: TCP
Raw
grafana.yaml
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2,
"links": [],
"panels": [
{
"datasource": {},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 40,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 7,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "edl2s4rg3zd34d"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(service) (rate(otelcol_receiver_accepted_spans{service=\"loadbalancer-collector-monitoring\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Received by the Load Balancer",
"type": "timeseries"
},
{
"datasource": {},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 40,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 7,
"x": 7,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "edl2s4rg3zd34d"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(service) (rate(otelcol_receiver_accepted_spans{service=\"backends-collector-monitoring\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Received by the Backends",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "fdl2v0peii7eod"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 7,
"x": 14,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.4.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "edl2s4rg3zd34d"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "otelcol_loadbalancer_num_backends{service=\"loadbalancer-collector-monitoring\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Number of backends",
"type": "stat"
},
{
"datasource": {},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 40,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 7,
"x": 0,
"y": 9
},
"id": 5,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "edl2s4rg3zd34d"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(service) (rate(otelcol_exporter_sent_spans{service=\"loadbalancer-collector-monitoring\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Exported by the Load Balancer",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "fdl2v0peii7eod"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 40,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 7,
"x": 7,
"y": 9
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "fdl2v0peii7eod"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "rate(otelcol_receiver_refused_spans{service=\"loadbalancer-collector-monitoring\"}[$__rate_interval])",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Rejected by the Load Balancer",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "fdl2v0peii7eod"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 7,
"x": 14,
"y": 9
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "edl2s4rg3zd34d"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(service) (rate(otelcol_loadbalancer_num_resolutions{service=\"loadbalancer-collector-monitoring\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Rate of Resolutions",
"type": "timeseries"
}
],
"refresh": "",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Load-balancer",
"uid": "fdl2tsqhygzy8a",
"version": 3,
"weekStart": ""
}
Raw
script.sh
k3d cluster create
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.yaml
kubectl wait --for=condition=Available deployments/cert-manager -n cert-manager
kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml
kubectl wait --for=condition=Available deployments/opentelemetry-operator-controller-manager -n opentelemetry-operator-system
kubectl create -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.73.2/bundle.yaml
kubectl wait --for=condition=Available deployments/prometheus-operator -n default
kubectl apply -f cr-load-balancer.yaml
kubectl wait --for=condition=Available deployments/loadbalancer-collector -n observability
kubectl wait --for=condition=Available deployments/backends-collector -n observability
kubectl port-forward -n observability service/prometheus-operated 9090:9090
kubectl port-forward -n observability service/loadbalancer-collector 4317:4317
kubectl port-forward -n observability deployments/grafana 3000:3000
# Add a new Prometheus datasource on Grafana, http://prometheus-operated:9090
telemetrygen traces --otlp-insecure --rate 1000 --duration 30m > telemetrygen.log 2>&1 &
sleep 5m
kubectl patch -n observability otelcol backends -p '{"spec":{"replicas":10}}' --type=merge
sleep 5m
kubectl patch -n observability otelcol backends -p '{"spec":{"replicas":5}}' --type=merge
sleep 5m
kubectl patch -n observability otelcol backends -p '{"spec":{"replicas":0}}' --type=merge
sleep 5m
kubectl patch -n observability otelcol backends -p '{"spec":{"replicas":5}}' --type=merge
sleep 5m
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment