Prometheus configuration for basic scraping and remote write.

prom.yml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
    name: prometheus
rules:
    - apiGroups: ['']
      resources:
          - nodes
          - nodes/proxy
          - services
          - endpoints
          - pods
      verbs: ['get', 'list', 'watch']
    - nonResourceURLs: ['/metrics', '/api']
      verbs: ['get']
---
apiVersion: v1
kind: ServiceAccount
metadata:
    name: prometheus
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
    name: prometheus
roleRef:
    apiGroup: rbac.authorization.k8s.io
    kind: ClusterRole
    name: prometheus
subjects:
    - kind: ServiceAccount
      name: prometheus
      namespace: default
---
apiVersion: apps/v1
kind: Deployment
metadata:
    name: prometheus
spec:
    replicas: 1
    selector:
        matchLabels:
            name: prometheus
    template:
        metadata:
            labels:
                name: prometheus
        spec:
            serviceAccountName: prometheus
            containers:
                - name: retrieval
                  image: 647662420899.dkr.ecr.us-east-1.amazonaws.com/paigo-agent:latest
                  args:
                      - '--config.file=/etc/prometheus/prometheus.yml'
                      - '--storage.tsdb.path=/prometheus'
                  imagePullPolicy: Always
                  ports:
                      - containerPort: 9090
                  volumeMounts:
                      - name: config-volume
                        mountPath: /etc/prometheus
                      - name: tenant-auth-token-default
                        mountPath: /var/run/default-tenant
                        readOnly: true
            volumes:
                - name: config-volume
                  configMap:
                      name: prometheus-config
                - name: tenant-auth-token-default
                  secret:
                      secretName: tenant-auth-token-default

---
kind: ConfigMap
apiVersion: v1
metadata:
    name: prometheus-config
data:
    prometheus.yml: |-
        global:
            scrape_interval: 1m
            scrape_timeout: 10s
            evaluation_interval: 1m
        scrape_configs:
        - job_name: kubernetes-nodes-cadvisor
          honor_timestamps: true
          scrape_interval: 1m
          scrape_timeout: 10s
          metrics_path: /metrics
          scheme: https
          authorization:
            type: Bearer
            credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
          tls_config:
            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
            insecure_skip_verify: true
          follow_redirects: true
          enable_http2: true
          relabel_configs:
          - separator: ;
            regex: __meta_kubernetes_node_label_(.+)
            replacement: $1
            action: labelmap
          - separator: ;
            regex: (.*)
            target_label: __address__
            replacement: kubernetes.default.svc:443
            action: replace
          - source_labels: [__meta_kubernetes_node_name]
            separator: ;
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
            action: replace
          metric_relabel_configs:
            - source_labels: [__name__]
              regex: container_.*
              action: keep
          kubernetes_sd_configs:
          - role: node
            kubeconfig_file: ""
            follow_redirects: true
            enable_http2: true
        remote_write:
          - url: http://xxxxx.us-east-1.elb.amazonaws.com/
            remote_timeout: 30s
            follow_redirects: true
            enable_http2: true
            queue_config:
              capacity: 2500
              max_shards: 200
              min_shards: 1
              max_samples_per_send: 10
              batch_send_deadline: 5s
              min_backoff: 30ms
              max_backoff: 5s
            metadata_config:
              send: true
              send_interval: 1m
              max_samples_per_send: 10