colinsurprenant

input { stdin {} }

filter {
  if [field1a] == "cond1a" {
    drop {}
  }
  if [field1a] == "cond1b" {
    drop {}
  }
  if [field1a] == "cond1c" {

colinsurprenant

# this is an alternate compact function implementation which also removes keys with empty string values
# writing the tests for this is left as an excercise to the reader :D

def compact(h)
  h.inject({}) do |result, (k, v)|
    if v.is_a?(Hash)
      result[k] = compact(v)
    elsif v.is_a?(String)
      result[k] = v unless v.empty?
    elsif !v.nil?

colinsurprenant

require "net/http"
require "json"
require "thread"

Thread.abort_on_exception = true

queue = Queue.new
POLL_DELAY = 1 # in seconds

poller_thread = Thread.new do

colinsurprenant

require "logstash/util/wrapped_acked_queue"
require "logstash/event"
require "logstash/instrument/namespaced_metric"


RSpec.configure do |config|
  config.backtrace_exclusion_patterns << /rakelib\/test/
  config.backtrace_exclusion_patterns << /vendor\/bundle/
  # config.order = :random
  # config.seed = 24307

colinsurprenant

rm -f data/queue/*
echo "test1" | bin/logstash  -e 'input{stdin{}} filter{} output{stdout{codec => rubydebug}}'
echo -e "\c" | bin/logstash  -e 'input{stdin{}} filter{} output{stdout{codec => rubydebug}}'
echo "test2" | bin/logstash  -e 'input{stdin{}} filter{} output{stdout{codec => rubydebug}}'
bin/cpdump checkpoint.head
hexdump data/queue/page.1 | head
echo -e "\c" | bin/logstash  -e 'input{stdin{}} filter{} output{stdout{codec => rubydebug}}'

colinsurprenant

or n in {1..200}; do echo -e "$n-1\n$n-2\n$n-3\n$n-4\n$n-5\n$n-6\n$n-7\n$n-8" | bin/logstash -w 4 -f fingerprint.conf;  done

colinsurprenant

$ while true; do cat syslog.dat; done | bin/logstash  -f syslog.conf | pv -Wbart > /dev/null

colinsurprenant

Measuring logstash filters performance

• given a data file sample named data.log
• modify your config to use the stdin input so that you can pipe you sample log file to logstash

input {
  stdin {
    codec => line
  }
}

colinsurprenant

require "thread"
require "benchmark"

COUNT = 50000000

@m1 = @m2 = @m3 = @m4  = @m5 = @m6 = @m7 = @m8 = @m9 = @m10 = @m11 = @m12 = nil
@m4 = "foo"
@m8 = "bar"

@calls = 0

colinsurprenant

Git/Github step-by-step Workflow

Step-by-step guide for creating a feature or bugfix branch, submit it for code review as a pull request and once approved, merge upstream. This guide is intended for internal developers with push access to all relevant repos.

You should understand rebasing and squashing. For a very good explanation on rebasing and squashing with pull requests see How to Rebase a Pull Request. Also worth reading is the Hacker's Guide to Git.

Setup