Skip to content

Instantly share code, notes, and snippets.

@axamon
Created February 17, 2016 22:12
Show Gist options
  • Save axamon/9a0fdf53ac302df9e46d to your computer and use it in GitHub Desktop.
Save axamon/9a0fdf53ac302df9e46d to your computer and use it in GitHub Desktop.
Utilizzo di gzip in python
def elabora():
"prende uno per uno i log ingestati e li elabora"
csv_filename = rlocal.brpoplpush("listalogdaelaborare","listalogelaborati",0)
csv_filename_parsed=csv_filename.strip().split("_")
#print "Sto elabrando il log: ",csv_filename
#f=gzip.open('/root/logs/archived-streamer-output/'+oggi+'/'+csv_filename, 'rU')
with gzip.open('/home/evangelion/logs/'+csv_filename_parsed[4]+'/'+csv_filename, 'rU') as f:
#salta la prima riga del log
next(f)
reader = csv.DictReader(f, delimiter="\t")
fieldnames = reader.fieldnames
try:
for line in reader:
#esegue solo per errori 40x e 50x
if bool(re.search("(403)|(5..)",str(line['request-desc/response-status']))) == True:
timestamp2=datetime.strptime(str(line['time-recvd(millisecond)']), "[%d/%b/%Y:%H:%M:%S.%f+0000]")
del line['time-recvd(millisecond)']
timestamp=datetime.isoformat(timestamp2)
line['timestamp']=str(timestamp)
line['ipcache']=str(csv_filename_parsed[3])
#print line
res= es.index(index="cdnerr-"+csv_filename_parsed[4], id=str(uuid.uuid4()), doc_type="log", body= json.dumps(line))
else:
pass
except ValueError:
pass
f.close()
rlocal.sadd("logfiniti",csv_filename)
return;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment