Grafana/Loki/Tempo

alloy.yaml

discovery.docker "flog_scrape" {
	host             = "unix:///var/run/docker.sock"
	refresh_interval = "5s"
}

discovery.relabel "flog_scrape" {
	targets = []

	rule {
		source_labels = ["__meta_docker_container_name"]
		regex         = "/(.*)"
		target_label  = "container"
	}
}

loki.source.docker "flog_scrape" {
	host             = "unix:///var/run/docker.sock"
	targets          = discovery.docker.flog_scrape.targets
	forward_to       = [loki.write.default.receiver]
	relabel_rules    = discovery.relabel.flog_scrape.rules
	refresh_interval = "5s"
}

loki.write "default" {
	endpoint {
		url       = "http://gateway:3100/loki/api/v1/push"
		tenant_id = "tenant1"
	}
	external_labels = {}
}

docker-compose.yaml

services:
    read:
        container_name: l-read
        image: grafana/loki:latest
        command: "-config.file=/etc/loki/config.yaml -target=read"
        ports:
            - 3101:3100
            - 7946
            - 9095
        volumes:
            - ./grafana-loki.yaml:/etc/loki/config.yaml:ro
            - ./data/loki:/loki
        depends_on:
            - minio
        healthcheck:
            test: [ "CMD", "/usr/bin/loki", "-health" ]
            start_period: 30s
            interval: 10s
            timeout: 5s
            retries: 5
        networks: &loki-dns
            loki:
                aliases:
                    - loki
    
    write:
        container_name: l-write
        image: grafana/loki:latest
        command: "-config.file=/etc/loki/config.yaml -target=write"
        ports:
            - 3102:3100
            - 7946
            - 9095
        volumes:
            - ./grafana-loki.yaml:/etc/loki/config.yaml:ro
            - ./data/loki:/loki
        healthcheck:
            test: [ "CMD", "/usr/bin/loki", "-health" ]
            start_period: 30s
            interval: 10s
            timeout: 5s
            retries: 5
        depends_on:
            - minio
        networks:
            <<: *loki-dns
    
    alloy:
        container_name: l-alloy
        image: grafana/alloy:latest
        volumes:
            - /var/run/docker.sock:/var/run/docker.sock
            - ./alloy.yaml:/etc/alloy/config.alloy:ro
            - ./data/alloy:/var/lib/alloy/data
        command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
        ports:
            - 12345:12345
        depends_on:
            - gateway
        networks:
            - loki
    
    minio:
        container_name: l-minio
        image: minio/minio
        entrypoint:
            - sh
            - -euc
            - |
                mkdir -p /data/loki-data && \
                mkdir -p /data/loki-ruler && \
                minio server /data
        environment:
            - MINIO_ROOT_USER=loki
            - MINIO_ROOT_PASSWORD=supersecret
            - MINIO_PROMETHEUS_AUTH_TYPE=public
            - MINIO_UPDATE=off
        ports:
            - 9000
        volumes:
            - ./data/minio:/data
        healthcheck:
            test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
            interval: 15s
            timeout: 20s
            retries: 5
        networks:
            - loki
    
    grafana:
        container_name: l-grafana
        image: grafana/grafana:latest
        environment:
            - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
            - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor traceQLStreaming
            - GF_DATE_FORMATS_DEFAULT_WEEK_START=monday
            - GF_USERS_WEEK_START=monday
            - TELEGRAM_BOT_TOKEN=XXXXXXXX:xxxxxxxxxxxxxxxxx
            - TELEGRAM_CHAT_ID="55xxxxxxx"
        depends_on:
            - gateway
            - prometheus
            - tempo
        volumes:
            - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/ds.yaml:ro
            - ./grafana-alerting.yaml:/etc/grafana/provisioning/alerting/contact-points.yaml:ro
            - ./data/grafana:/var/lib/grafana
        ports:
            - "3000:3000"
        healthcheck:
            test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1" ]
            interval: 10s
            timeout: 5s
            retries: 5
        networks:
            - loki
    
    backend:
        container_name: l-loki
        image: grafana/loki:latest
        volumes:
            - ./grafana-loki.yaml:/etc/loki/config.yaml:ro
            - ./data/loki:/loki
        ports:
            - "3100"
            - "7946"
        command: "-config.file=/etc/loki/config.yaml -target=backend -legacy-read-mode=false"
        depends_on:
            - gateway
        healthcheck:
            test: [ "CMD", "/usr/bin/loki", "-health" ]
            start_period: 30s
            interval: 30s
            timeout: 10s
            retries: 5
        networks:
            - loki
    
    gateway:
        container_name: l-gateway
        image: nginx:latest
        depends_on:
            - read
            - write
        volumes:
            - ./nginx.conf:/etc/nginx/nginx.conf:ro
        ports:
            - "3100:3100"
        healthcheck:
            test: [ "CMD", "service", "nginx", "status" ]
            interval: 10s
            timeout: 5s
            retries: 5
        networks:
            - loki
    
    flog:
        container_name: l-flog
        image: mingrammer/flog
        command: -f json -d 200ms -l
        networks:
            - loki
    
    prometheus:
        container_name: l-prometheus
        image: prom/prometheus:latest
        command:
            - --config.file=/etc/prometheus/prometheus.yml
            - --storage.tsdb.path=/prometheus
            - --web.enable-otlp-receiver
            - --web.enable-remote-write-receiver
            - --enable-feature=exemplar-storage
            - --enable-feature=native-histograms
        volumes:
            - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
            - ./data/prometheus:/prometheus
        ports:
            - "9090:9090"
        healthcheck:
            test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
            interval: 10s
            timeout: 5s
            retries: 5
        networks:
            - loki
    
    tempo:
        container_name: l-tempo
        image: grafana/tempo:2.7.1
        command: [ "-config.file=/etc/tempo.yaml" ]
        volumes:
            - ./tempo.yaml:/etc/tempo.yaml:ro
            - ./data/tempo:/var/tempo
        ports:
            - "3200:3200"
            - "4317:4317"
            - "4318:4318"
        networks:
            - loki

networks:
    loki:

grafana-alerting.yaml

apiVersion: 1

contactPoints:
  - orgId: 1
    name: telegram-main
    receivers:
      - uid: telegram-main-receiver
        type: telegram
        settings:
          bottoken: $TELEGRAM_BOT_TOKEN
          chatid: $TELEGRAM_CHAT_ID
          parse_mode: HTML
          message: |
            <b>{{ .Status | toUpper }}</b> - {{ .CommonLabels.alertname }}
            {{ if .CommonLabels.service }}<b>Service:</b> {{ .CommonLabels.service }}{{ end }}
            {{ if .CommonLabels.severity }}<b>Severity:</b> {{ .CommonLabels.severity }}{{ end }}

            {{ range .Alerts }}
            <b>{{ .Annotations.summary }}</b>
            {{ .Annotations.description }}

            {{ if .Annotations.runbook_url }}<a href="{{ .Annotations.runbook_url }}">Runbook</a>{{ end }}
            {{ end }}
        disableResolveMessage: false

policies:
  - orgId: 1
    receiver: telegram-main
    group_by: ['alertname', 'service']
    group_wait: 30s
    group_interval: 5m
    repeat_interval: 4h
    routes:
      - receiver: telegram-main
        matchers:
          - severity = critical
        group_wait: 0s
        repeat_interval: 1h

grafana-datasources.yaml

apiVersion: 1
datasources:
  - name: Loki
    type: loki
    uid: loki
    access: proxy
    url: http://gateway:3100
    jsonData:
      httpHeaderName1: X-Scope-OrgID
      derivedFields:
        - name: TraceID
          matcherRegex: '"TraceId":"([a-f0-9]+)"'
          url: '${__value.raw}'
          datasourceUid: tempo
    secureJsonData:
      httpHeaderValue1: default

  - name: Prometheus
    type: prometheus
    uid: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    jsonData:
      timeInterval: 15s
      exemplarTraceIdDestinations:
        - name: trace_id
          datasourceUid: tempo

  - name: Tempo
    type: tempo
    uid: tempo
    access: proxy
    url: http://tempo:3200
    jsonData:
      tracesToLogsV2:
        datasourceUid: loki
        spanStartTimeShift: -5m
        spanEndTimeShift: 5m
        filterByTraceID: true
        tags:
          - key: service.name
            value: service
      tracesToMetrics:
        datasourceUid: prometheus
        tags:
          - key: service.name
            value: service_name
      serviceMap:
        datasourceUid: prometheus
      nodeGraph:
        enabled: true

grafana-loki.yaml

server:
  http_listen_address: 0.0.0.0
  http_listen_port: 3100

memberlist:
  join_members: ["read", "write", "backend"]
  dead_node_reclaim_time: 30s
  gossip_to_dead_nodes_time: 15s
  left_ingesters_timeout: 30s
  bind_addr: ['0.0.0.0']
  bind_port: 7946
  gossip_interval: 2s

schema_config:
  configs:
    - from: 2023-01-01
      store: tsdb
      object_store: s3
      schema: v13
      index:
        prefix: index_
        period: 24h
common:
  path_prefix: /loki
  replication_factor: 1
  compactor_address: http://backend:3100
  storage:
    s3:
      endpoint: minio:9000
      insecure: true
      bucketnames: loki-data
      access_key_id: loki
      secret_access_key: supersecret
      s3forcepathstyle: true
  ring:
    kvstore:
      store: memberlist
ruler:
  storage:
    s3:
      bucketnames: loki-ruler

compactor:
  working_directory: /tmp/compactor

limits_config:
  allow_structured_metadata: true

storage_config:
  tsdb_shipper:
    active_index_directory: /loki/tsdb-index
    cache_location: /loki/tsdb-cache

nginx.conf

user  nginx;
worker_processes  5;

events {
  worker_connections  1000;
}

http {
  resolver 127.0.0.11;

  server {
    listen 3100;

    location = / {
      return 200 'OK';
      auth_basic off;
    }

    location = /api/prom/push {
      proxy_pass http://write:3100$request_uri;
    }

    location = /api/prom/tail {
      proxy_pass http://read:3100$request_uri;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "upgrade";
    }

    location ~ /api/prom/.* {
      proxy_pass http://read:3100$request_uri;
    }

    location = /loki/api/v1/push {
      proxy_set_header X-Scope-OrgID "default";
      proxy_pass http://write:3100$request_uri;
    }

    location = /loki/api/v1/tail {
      proxy_pass http://read:3100$request_uri;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "upgrade";
    }

    location ~ /loki/api/.* {
      proxy_pass http://read:3100$request_uri;
    }
  }
}

prometheus.yml

global:
  scrape_interval: 10s
  evaluation_interval: 10s

otlp:
  promote_resource_attributes:
    - service.name
    - service.namespace
    - service.instance.id
    - service.version
    - deployment.environment
    - host.name
  translation_strategy: NoUTF8EscapingWithSuffixes

storage:
  tsdb:
    out_of_order_time_window: 30m

tempo.yaml

server:
  http_listen_port: 3200

distributor:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318

ingester:
  max_block_duration: 5m

compactor:
  compaction:
    block_retention: 168h

storage:
  trace:
    backend: local
    wal:
      path: /var/tempo/wal
    local:
      path: /var/tempo/blocks

metrics_generator:
  registry:
    external_labels:
      source: tempo
  storage:
    path: /var/tempo/generator/wal
    remote_write:
      - url: http://prometheus:9090/api/v1/write
        send_exemplars: true
  traces_storage:
    path: /var/tempo/generator/traces
  processor:
    local_blocks:
      filter_server_spans: false
      flush_to_storage: true

overrides:
  defaults:
    metrics_generator:
      processors: [service-graphs, span-metrics, local-blocks]