Python deduplicating log filter

logging_config.py

import logging
import atexit
import pprint


class DeduplicationFilter(logging.Filter):
    def __init__(self):
        self.last_record = None
        self.count = 1
        atexit.register(self._flush)

    def _simplify_record(self, record):
        keys = ['msg', 'levelname', 'name', 'lineno', 'funcName', 'levelno']
        return {k: getattr(record, k) for k in keys}

    def filter(self, record):
        simple_record = self._simplify_record(record)
        if self.last_record == simple_record:
            # duplicate
            self.count += 1
            return False
        # not duplicate
        # pprint.pprint(record.__dict__)  
        self._flush()
        self.last_record = simple_record
        return True

    def _flush(self):
        if self.count > 1:
            self.last_record[
                'msg'] = f"last message repeated {self.count} times"
            _log.handle(logging.makeLogRecord(self.last_record))
            self.count = 1


_log = logging.getLogger()

FILTER = DeduplicationFilter()

main.py

import sys
import logging
from logging_config import FILTER


log = logging.getLogger(__name__)

logfmt_str = "%(asctime)s %(levelname)-8s pid:%(process)d %(name)s:%(lineno)03d:%(funcName)s %(message)s"
logging.basicConfig(format=logfmt_str, level=logging.DEBUG)
log.addFilter(FILTER)

def main():
  log.debug("begining of main")  
  for _ in range(10):
    log.warning("test_function message ten")

  log.warning("test_function message one")

  for _ in range(200):
    log.info("test_function message two hundred")

  log.debug("end of main")

  return 0


if __name__ == '__main__':
  sys.exit(main())