Go program to parse AWS ELB access logs to Go structs and then output as JSONL

awselbtojson.go

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Quick hack to turn an ELB access log into a Go struct, and then into JSON.
// From there you can query with JSONPath using https://github.com/ohler55/ojg

type ELBLogEntry struct {
	Type                   string  `json:"type"`
	Time                   string  `json:"time"`
	ELB                    string  `json:"elb"`
	ClientPort             string  `json:"client_port"`
	TargetPort             string  `json:"target_port"`
	RequestProcessingTime  float64 `json:"request_processing_time"`
	TargetProcessingTime   float64 `json:"target_processing_time"`
	ResponseProcessingTime float64 `json:"response_processing_time"`
	ELBStatusCode          string  `json:"elb_status_code"`
	TargetStatusCode       string  `json:"target_status_code"`
	ReceivedBytes          int64   `json:"received_bytes"`
	SentBytes              int64   `json:"sent_bytes"`
	Request                string  `json:"request"`
	UserAgent              string  `json:"user_agent"`
	SSLCipher              string  `json:"ssl_cipher"`
	SSLProtocol            string  `json:"ssl_protocol"`
	TargetGroupARN         string  `json:"target_group_arn"`
	TraceID                string  `json:"trace_id"`
	DomainName             string  `json:"domain_name"`
	ChosenCertARN          string  `json:"chosen_cert_arn"`
	MatchedRulePriority    string  `json:"matched_rule_priority"`
	RequestCreationTime    string  `json:"request_creation_time"`
	ActionsExecuted        string  `json:"actions_executed"`
	RedirectURL            string  `json:"redirect_url"`
	ErrorReason            string  `json:"error_reason"`
	TargetPortListened     string  `json:"target_port_listened"`
	TargetStatusCodeList   string  `json:"target_status_code_list"`
	Classification         string  `json:"classification"`
	ClassificationReason   string  `json:"classification_reason"`
	ConnTraceID            string  `json:"conn_trace_id"`
}

var quotedRE = regexp.MustCompile(`^"([^"]+)"\s*`)

var unquotedRE = regexp.MustCompile(`^([^ ]+)\s*`)

func nextToken(line string) (string, string, error) {
	var chunks []string
	chunks = quotedRE.FindStringSubmatch(line)
	if chunks != nil {
		tok := chunks[1]
		rest := line[len(chunks[0]):]
		return tok, rest, nil
	}
	chunks = unquotedRE.FindStringSubmatch(line)
	if chunks != nil {
		tok := chunks[1]
		rest := line[len(chunks[0]):]
		return tok, rest, nil
	}
	return "", line, fmt.Errorf("no next token found in [%s]", line)
}

func splitLogEntry(line string) []string {
	var result []string
	for {
		tok, rest, err := nextToken(line)
		if err != nil {
			return result
		}
		result = append(result, tok)
		line = rest
	}
}

func parseELBLogEntry(line string) (*ELBLogEntry, error) {
	cols := splitLogEntry(line)
	if len(cols) < 15 {
		return nil, fmt.Errorf("not enough columns in log entry: %s", line)
	}
	return &ELBLogEntry{
		Type:                   cols[0],
		Time:                   cols[1],
		ELB:                    cols[2],
		ClientPort:             cols[3],
		TargetPort:             cols[4],
		RequestProcessingTime:  parseFloat(cols[5]),
		TargetProcessingTime:   parseFloat(cols[6]),
		ResponseProcessingTime: parseFloat(cols[7]),
		ELBStatusCode:          cols[8],
		TargetStatusCode:       cols[9],
		ReceivedBytes:          parseInt64(cols[10]),
		SentBytes:              parseInt64(cols[11]),
		Request:                cols[12],
		UserAgent:              cols[13],
		SSLCipher:              cols[14],
		SSLProtocol:            cols[15],
		TargetGroupARN:         cols[16],
		TraceID:                cols[17],
		DomainName:             cols[18],
		ChosenCertARN:          cols[19],
		MatchedRulePriority:    cols[20],
		RequestCreationTime:    cols[21],
		ActionsExecuted:        cols[22],
		RedirectURL:            cols[23],
		ErrorReason:            cols[24],
		TargetPortListened:     cols[25],
		TargetStatusCodeList:   cols[26],
		Classification:         cols[27],
		ClassificationReason:   cols[28],
		ConnTraceID:            cols[29],
	}, nil
}

func parseFloat(s string) float64 {
	var f float64
	_, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return 0.0
	}
	return f
}

func parseInt64(s string) int64 {
	var i int64
	_, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		return 0
	}
	return i
}

func main() {
	rdr := bufio.NewScanner(os.Stdin)
	for rdr.Scan() {
		line := rdr.Text()
		if line == "" {
			continue
		}
		entry, err := parseELBLogEntry(line)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing log entry: %v\n", err)
			fmt.Fprintf(os.Stderr, "Line: %s\n", line)
			os.Exit(1)
		}
		if !strings.HasPrefix(entry.ELBStatusCode, "5") {
			continue
		}
		jd, err := json.MarshalIndent(entry, "", "  ")
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error marshaling log entry to JSON: %v\n", err)
			os.Exit(1)
		}
		fmt.Println(string(jd))
	}
}

awselbtojson_test.go

package main

import "testing"

func TestNextToken(t *testing.T) {
	tests := []struct {
		line string
		tok  string
		rest string
	}{
		{`hello "world"`, "hello", `"world"`},
		{`"hello world"`, "hello world", ""},
		{`"hello world" foo`, "hello world", "foo"},
		{`hello world`, "hello", "world"},
	}

	for _, test := range tests {
		tok, rest, err := nextToken(test.line)
		if err != nil {
			t.Errorf("nextToken(%q) returned error: %v", test.line, err)
			continue
		}
		if tok != test.tok || rest != test.rest {
			t.Errorf("nextToken(%s) = (%s, %s), want (%s, %s)", test.line, tok, rest, test.tok, test.rest)
		}
	}
}