ingest.py

import csv
import sys
import os
import gzip
import datetime
import pyes
from pyes import *

now = datetime.datetime.now()

fieldnames=["@timestamp","tts","ipclient","reqest-response","bytes","method","url","mime","ua"]

csv_filename = sys.argv[1]

print "Opening GZ file: ",csv_filename 

f=gzip.open(csv_filename, 'rb')
csv_reader = csv.DictReader(f,fieldnames,delimiter='\t')

index = "cdnlogs-"+now.strftime("%Y-%m-%d")

conn = ES('127.0.0.1:9200') #Use HTTP

for line in csv_reader:
        #decommenta per vedere a video
        #print line
        conn.index(line,index,"cdnlogs", bulk=True)
f.close()
os.remove(csv_filename)