Created
February 17, 2016 22:12
-
-
Save axamon/9a0fdf53ac302df9e46d to your computer and use it in GitHub Desktop.
Utilizzo di gzip in python
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def elabora(): | |
"prende uno per uno i log ingestati e li elabora" | |
csv_filename = rlocal.brpoplpush("listalogdaelaborare","listalogelaborati",0) | |
csv_filename_parsed=csv_filename.strip().split("_") | |
#print "Sto elabrando il log: ",csv_filename | |
#f=gzip.open('/root/logs/archived-streamer-output/'+oggi+'/'+csv_filename, 'rU') | |
with gzip.open('/home/evangelion/logs/'+csv_filename_parsed[4]+'/'+csv_filename, 'rU') as f: | |
#salta la prima riga del log | |
next(f) | |
reader = csv.DictReader(f, delimiter="\t") | |
fieldnames = reader.fieldnames | |
try: | |
for line in reader: | |
#esegue solo per errori 40x e 50x | |
if bool(re.search("(403)|(5..)",str(line['request-desc/response-status']))) == True: | |
timestamp2=datetime.strptime(str(line['time-recvd(millisecond)']), "[%d/%b/%Y:%H:%M:%S.%f+0000]") | |
del line['time-recvd(millisecond)'] | |
timestamp=datetime.isoformat(timestamp2) | |
line['timestamp']=str(timestamp) | |
line['ipcache']=str(csv_filename_parsed[3]) | |
#print line | |
res= es.index(index="cdnerr-"+csv_filename_parsed[4], id=str(uuid.uuid4()), doc_type="log", body= json.dumps(line)) | |
else: | |
pass | |
except ValueError: | |
pass | |
f.close() | |
rlocal.sadd("logfiniti",csv_filename) | |
return; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment