Local logs to GCL

otelcol.yaml

receivers:
  filelog/tcp:
    include: [ /home/jpkroehling/Downloads/tmp/*.log ]
    storage: file_storage/filelogreceivertcp
    operators:
      - type: regex_parser
        regex: '^(?P<timestamp>[^ ]* [^ ]*) (?P<process>[^[]*)[[](?P<process_id>[^]]*)[]][[](?P<level>[^]]*)[]] (?P<message>.*)$'
        timestamp:
          parse_from: attributes.timestamp
          layout: '%Y-%m-%d %H.%M.%S.%L'
    start_at: beginning
    attributes:
      - source: tcp

processors:
  batch:
    timeout: 10s

  attributes/tcp:
    actions:
      - action: insert
        key: loki.attribute.labels
        value: process, process_id, source, level, station_id, filename

  transform/tcp:
    error_mode: propagate
    log_statements:
      - context: log
        statements:
         - merge_maps(attributes,ExtractPatterns(attributes["log.file.name"],"^(?P<station_id>[0-9]{9})"),"upsert")
         - replace_pattern(body,"([A-H|J-N|P|R-Z|0-9]{17})","$$1",SHA256)
         - replace_pattern(attributes["log.file.name"],"^([0-9]{9})","$$1",SHA256)
         - replace_pattern(attributes["station_id"],"^([0-9]{9})","$$1",SHA256)
         - replace_all_patterns(attributes, "key", "log.file.name", "filename")
         - delete_key(attributes,"message")

exporters:
  logging:
    verbosity: basic
    
  loki/remote:
    endpoint: "https://logs-prod-us-central1.grafana.net/loki/api/v1/push"
    auth:
      authenticator: basicauth/loki
    default_labels_enabled:
      exporter: false
      level: false
      job: false

extensions:
  file_storage/filelogreceivertcp:
    directory: /home/jpkroehling/Downloads/tmp/receiver

  basicauth/loki:
    client_auth:
      username: "87940"
      password: "eyJrIjoiN2RjYTZlOGFhYTQ3ZmFhOTkyODJhZGVjYzk5MzE0Y2VmZGU1MzVmZSIsIm4iOiJsb2tpLTIwMjQtMDQtMjQiLCJpZCI6NTI5MzcxfQ=="

service:
  extensions: [basicauth/loki,file_storage/filelogreceivertcp]
  pipelines:
    logs/tcp:
      receivers: [filelog/tcp]
      processors: [batch,transform/tcp,attributes/tcp]
      exporters: [logging,loki/remote]