Process a WordPress log file (debug.log) into a tally of how frequently each entry appears

prioritise-logs.py

import re
from pandas import DataFrame

# Replace with your file names
read_file = 'debug.log'
write_file = 'output.csv'

# Utility vars
count = []
message = []
percent= []
total = 0

# Populate tally of log entries
with open(read_file) as f1:
    d = {} # Dictionary for tally
    lines = f1.readlines()
    for line in lines:
        total += 1
        # Remove date from start of line
        line = re.sub(r"\[[^\]]+\]\s?(.*)", r"\1", line)
        # Remove trailing \n
        line = line.rstrip("\n")
        if line in d:
            d[line] += 1
        else:
            d[line] = 1

# Populate lists for DataFrame columns
for key in d:
    message.append(key)
    count.append(d[key])
    percent.append(round(d[key]/total, 5))

# Create DataFrame with headings and data
df = DataFrame({'count': count, 'percent': percent, 'message': message})
df.sort_values(['count'], inplace=True, ascending=False)
df.to_csv(write_file, index=False)