Demonstrating logs data stream default and customizable mappings

logs-mappings-demo.http

### Create a logs-*-* data stream
//@no-log
PUT {{host}}/_data_stream/logs-generic-default
Authorization: Basic {{username}} {{password}}
Content-Type: multipart/form-data; boundary=WebAppBoundary

### Index a document
//@no-log
POST {{host}}/logs-generic-default/_doc
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "message": "invalid_ip",
  "test": {
    "valid_ip": "127.0.0.1",
    "invalid_ip": "invalid"
  }
}

### Search for a document
//@no-log
POST {{host}}/logs-generic-default/_search
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "query": {
    "term": {
      "message": {
        "value": "invalid_ip"
      }
    }
  },
  "fields": [
    "test.invalid_ip"
  ]
}

### Get backing index mapping
//@no-log
GET {{host}}/logs-generic-default/_mapping
Authorization: Basic {{username}} {{password}}

### Add custom mapping for keyword IP field
//@no-log
PUT {{host}}/_component_template/logs@custom
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "template": {
    "mappings": {
      "properties": {
        "test.keyword_ip": {
          "type": "keyword"
        }
      }
    }
  }
}

### Rollover my data-stream to get the new custom mapping applied
//@no-log
POST {{host}}/logs-generic-default/_rollover/
Authorization: Basic {{username}} {{password}}

### Index a document with custom mappings
//@no-log
POST {{host}}/logs-generic-default/_doc
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "message": "keyword_ip",
  "test": {
    "valid_ip": "127.0.0.1",
    "keyword_ip": "invalid"
  }
}

### Search for the document
//@no-log
POST {{host}}/logs-generic-default/_search
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "query": {
    "term": {
      "message": {
        "value": "keyword_ip"
      }
    }
  },
  "fields": [
    "test.keyword_ip"
  ]
}

### Add custom pipeline for invalid_ip field
//@no-log
PUT {{host}}/_ingest/pipeline/logs@custom
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "description" : "Renaming a keyword IP field to one with proper mapping",
  "processors" : [
    {
      "rename" : {
        "field": "test.invalid_ip",
        "target_field": "test.keyword_ip",
        "ignore_missing": true
      }
    }
  ]
}

### Add custom pipeline for automatic JSOM message parsing
//@no-log
PUT {{host}}/_ingest/pipeline/logs@custom
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "description" : "Renaming a keyword IP field to one with proper mapping",
  "processors" : [
    {
      "pipeline" : {
        "name": "logs@json-message",
        "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such"
      }
    }
  ]
}

### Index a document with JSON-formatted message
//@no-log
POST {{host}}/logs-generic-default/_doc
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "@timestamp": "2023-06-14",
  "message": "{\"message\":\"extracted_message\",  \"log.level\": \"INFO\", \"service.name\":\"my-app\"}"
}

### Search for the document
//@no-log
POST {{host}}/logs-generic-default/_search
Authorization: Basic {{username}} {{password}}
Content-Type: application/json

{
  "query": {
    "term": {
      "message": {
        "value": "extracted_message"
      }
    }
  },
  "fields": ["*"]
}