Ukázka načítání nfdump souboru pomocí knihovny BitStruct a BinData

nfdump_bindata.rb

require 'bindata'

class FlowFileHeader < BinData::Record
  endian :little
  uint16 :magic
  uint16 :version
  uint32 :flags
  uint32 :numBlocks
  
  string :ident, :read_length => 128
  
end

class StatRecord < BinData::Record
  endian :little
  #overall stat
  uint64  :numflows
  uint64  :numbytes
  uint64  :numpackets
  #flow stat
  uint64  :numflows_tcp
  uint64  :numflows_udp
  uint64  :numflows_icmp
  uint64  :numflows_other
  #bytes stat
  uint64  :numbytes_tcp
  uint64  :numbytes_udp
  uint64  :numbytes_icmp
  uint64  :numbytes_other
  #packet stat
  uint64  :numpackets_tcp
  uint64  :numpackets_udp
  uint64  :numpackets_icmp
  uint64  :numpackets_other
  #time window
  uint32  :first_seen
  uint32  :last_seen
  uint16  :msec_first
  uint16  :msec_last
  #other
  uint32  :sequence_failure
end

class DataBlockHeader < BinData::Record
  endian :little
  uint32 :numRecords   # number of data records in data block
  uint32 :blockSize     # size of this block in bytes without this header
  uint16 :id       # Block ID == DATA_BLOCK_TYPE_2
  # uint16 :pad      # unused align 32 bit
  skip :length => 2
end

class ExtensionMap < BinData::Record
  endian :little

  uint16 :type
  uint16 :mapSize, :value => lambda { restOfMap.length }
  
  uint16 :mapId
  uint16 :extensionSize
  array :ext_id, :type => :uint16, :initial_length => 1
  string :restOfMap, :read_length => lambda {mapSize-10}
  #uint32 :neco
  #skip :length => 6
end

class CommonRecord < BinData::Record
  endian :little
  
  #record head
  uint16 :type
  uint16 :recordSize
  
  #record meta data
  string :flag, :read_length => 1
  # uint8 :flag
  uint8 :exporter_ref
  uint16 :ext_map

  #netflow common record
  uint16 :msec_first
  uint16 :msec_last
  uint32 :firstRec
  uint32 :lastRec
 
  uint8 :fwd_status
  uint8 :tcp_flags
  uint8 :prot
  uint8 :tos
  uint16 :srcport
  uint16 :dstport

  uint32 :srcip
  uint32 :dstip
  uint32 :dOcts
  uint32 :dPkts
  uint8 :wtf1
  uint24 :wtg
  uint32 :wtf2
  #link to extensions
  #uint32  data[1]
  #array :data, :type => :uint32, :initial_length => 1
  #skip :length => 20
end

class OldRecord < BinData::Record
  endian :little
  
  uint32 :flags_o
  uint16 :recordSize_o
  uint16 :expRef_o
  uint16 :msec_first_o
  uint16 :msec_last_o
  uint32 :first_o
  uint32 :last_o
   
  uint8 :dir_o
  uint8 :tcp_flags_o
  uint8 :prot_o
  uint8 :tos_o
  uint16 :input_o
  uint16 :output_o
  uint16 :srcport_o
  uint16 :dstport_o
  uint16 :srcas_o
  uint16 :dstas_o
  array :data_o, :type => :uint8, :initial_length => 4
  # uint8 :data[4]
 
end
   
file=File.open("./../tmp/nfcapd.201111182320","r")

header=FlowFileHeader.read(file)
puts "file header"
puts header.inspect

fileStat=StatRecord.read(file)
puts "file stats"
puts fileStat.inspect

dataBlockHeader=DataBlockHeader.read(file)
puts "data block header"
puts dataBlockHeader.inspect

record1=OldRecord.read(file)
puts "rec1"
puts record1.inspect

record2=OldRecord.read(file)
puts "rec2"
puts record2.inspect

file=File.open("./../tmp/nfcapd.out.2","r")

header=FlowFileHeader.read(file)
puts "file header"
puts header.inspect

fileStat=StatRecord.read(file)
puts "file stats"
puts fileStat.inspect

dataBlockHeader=DataBlockHeader.read(file)
puts "data block header"
puts dataBlockHeader.inspect

record1=ExtensionMap.read(file)
puts "record1"
puts record1.inspect

record2=CommonRecord.read(file)
puts "record2"
puts record2.inspect

record3=CommonRecord.read(file)
puts "record3"
puts record3.inspect

nfdump_bitstruct.rb

require 'bit-struct'

class FlowFileHeader < BitStruct
  unsigned :magic, 16, "magic", :endian=>:little
  unsigned :version, 16, "version", :endian=>:little
  unsigned :flags, 32, "flags", :endian=>:little
  unsigned :numBlocks, 32, "numBlocks", :endian=>:little
  char     :ident, 1024, "ident", :endian=>:little
end



class StatRecord < BitStruct
  unsigned :numflows, 64, "numflows", :endian=>:little
  unsigned :numbytes, 64, "numbytes", :endian=>:little
  unsigned :numpackets, 64, "numpackets", :endian=>:little
  # flow stat
  unsigned :numflows_tcp, 64, "numflows_tcp", :endian=>:little
  unsigned :numflows_udp, 64, "numflows_udp", :endian=>:little
  unsigned :numflows_icmp, 64, "numflows_icmp", :endian=>:little
  unsigned :numflows_other, 64, "numflows_other", :endian=>:little
  # bytes stat
  unsigned :numbytes_tcp, 64, "numbytes_tcp", :endian=>:little
  unsigned :numbytes_udp, 64, "numbytes_udp", :endian=>:little
  unsigned :numbytes_icmp, 64, "numbytes_icmp", :endian=>:little
  unsigned :numbytes_other, 64, "numbytes_other", :endian=>:little
  # packet stat
  unsigned :numpackets_tcp, 64, "numpackets_tcp", :endian=>:little
  unsigned :numpackets_udp, 64, "numpackets_udp", :endian=>:little
  unsigned :numpackets_icmp, 64, "numpackets_icmp", :endian=>:little
  unsigned :numpackets_other, 64, "numpackets_other", :endian=>:little
  # time window
  unsigned :first_seen, 32, "first_seen", :endian=>:little
  unsigned :last_seen, 32, "last_seen", :endian=>:little
  unsigned :msec_first, 16, "msec_first", :endian=>:little
  unsigned :msec_last, 16, "msec_last", :endian=>:little
  # other
  unsigned :sequence_failure, 32, "sequence_failure", :endian=>:little
end



class HeadRecord < BitStruct
  unsigned :numRecords,32,"numRecords",:endian=>:little   # number of data records in data block
  unsigned :blockSize,32,"blockSize",:endian=>:little     # size of this block in bytes without this header
  unsigned :id ,16,"id",:endian=>:little      # Block ID == DATA_BLOCK_TYPE_2
  # uint16 :pad      # unused align 32 bit
  # skip :length => 2
  unsigned :pad,16,"pad",:endian=>:little
end

class OldRecord < BitStruct  
  unsigned :flags_o,32,"flags_o",:endian=>:little
  unsigned :recordSize_o,16,"recordSize_o",:endian=>:little
  unsigned :expRef_o,16,"expRef_o",:endian=>:little
  unsigned :msec_first_o,16,"msec_first_o",:endian=>:little
  unsigned :msec_last_o,16,"msec_last_o",:endian=>:little
  unsigned :first_o,32,"first_o",:endian=>:little
  unsigned :last_o,32,"last_o",:endian=>:little
   
  unsigned :dir_o,8,"dir_o",:endian=>:little
  unsigned :tcp_flags_o,8,"tcp_flags_o",:endian=>:little
  unsigned :prot_o,8,"prot_o",:endian=>:little
  unsigned :tos_o,8,"tos_o",:endian=>:little
  unsigned :input_o,16,"input_o",:endian=>:little
  unsigned :output_o,16,"output_o",:endian=>:little
  unsigned :srcport_o,16,"srcport_o",:endian=>:little
  unsigned :dstport_o,16,"dstport_o",:endian=>:little
  unsigned :srcas_o,16,"srcas_o",:endian=>:little
  unsigned :dstas_o,16,"dstas_o",:endian=>:little
  #array :data_o, :type => :uint8, :initial_length => 4
  # uint8 :data[4]
 
end

file=File.open("./../tmp/nfcapd.201111182320","r")

header=FlowFileHeader.new(file.read(140))

puts header.inspect_detailed

statRecord=StatRecord.new(file.read(136))

puts statRecord.inspect_detailed

puts HeadRecord.round_byte_length()

headRecord=HeadRecord.new(file.read(HeadRecord.round_byte_length()+1)) #why +1? compress?

puts headRecord.inspect_detailed

record=OldRecord.new(file.read(136))

puts record.inspect_detailed