vector_bulk_update.yaml

sources:
  generate_syslog:
    type:   "demo_logs"
    format: "syslog"
    count:  100

transforms:
  remap_syslog:
    inputs:
      - "generate_syslog"
    type:   "remap"
    source: |
            structured = parse_syslog!(.message)
            . = merge(., structured)
            .doc = .
            .doc_as_upsert = true
            .msgid =  sha1("foo")

sinks:
  emit_syslog:
    inputs:
      - "remap_syslog"
    type: "console"
    encoding:
      codec: "json"
  sink_elastic:
    type: "elasticsearch"
    mode: bulk
    inputs:
      - "remap_syslog"
    endpoint: http://localhost:9200/
    bulk:
      action : update
    encoding:
      only_fields:
        - "doc"
        - "doc_as_upsert"
    id_key:
      msgid