Created
June 4, 2015 11:35
-
-
Save danishabdullah/d756fab3eba0f333b32a to your computer and use it in GitHub Desktop.
Python logger for multiprocessing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import multiprocessing, threading, logging, sys, traceback | |
class MultiProcessingLog(logging.Handler): | |
def __init__(self, name, mode='w', maxsize=0, rotate=0): | |
logging.Handler.__init__(self) | |
self._handler = logging.handlers.RotatingFileHandler(name, mode, maxsize, rotate, encoding='utf-8') | |
self.queue = multiprocessing.Queue(-1) | |
t = threading.Thread(target=self.receive) | |
t.daemon = True | |
t.start() | |
def setFormatter(self, fmt): | |
logging.Handler.setFormatter(self, fmt) | |
self._handler.setFormatter(fmt) | |
def receive(self): | |
while True: | |
try: | |
record = self.queue.get() | |
self._handler.emit(record) | |
except (KeyboardInterrupt, SystemExit): | |
raise | |
except EOFError: | |
break | |
except: | |
traceback.print_exc(file=sys.stderr) | |
def send(self, s): | |
self.queue.put_nowait(s) | |
def _format_record(self, record): | |
# ensure that exc_info and args | |
# have been stringified. Removes any chance of | |
# unpickleable things inside and possibly reduces | |
# message size sent over the pipe | |
if record.args: | |
record.msg = record.msg % record.args | |
record.args = None | |
if record.exc_info: | |
dummy = self.format(record) | |
record.exc_info = None | |
return record | |
def emit(self, record): | |
try: | |
s = self._format_record(record) | |
self.send(s) | |
except (KeyboardInterrupt, SystemExit): | |
raise | |
except: | |
self.handleError(record) | |
def close(self): | |
self._handler.close() | |
logging.Handler.close(self) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment