parser anubis log, minimax version

parser_anubis_minimax.py

#!/usr/bin/env python3
import json
import sys
import os
import time
import tracemalloc
from collections import Counter
from datetime import datetime

def format_bytes(bytes_val):
    for unit in ['B', 'KB', 'MB', 'GB']:
        if bytes_val < 1024:
            return f"{bytes_val:.2f} {unit}"
        bytes_val /= 1024
    return f"{bytes_val:.2f} TB"

def parse_anubis_log(filepath):
    blocked_bots = Counter()
    blocked_pages = Counter()
    bot_block_techniques = {}
    first_time = None
    last_time = None
    line_count = 0

    tracemalloc.start()
    start_time = time.time()

    with open(filepath, 'r') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            entry = json.loads(line)
            line_count += 1

            timestamp = entry.get('time', '')
            msg = entry.get('msg', '')

            if first_time is None:
                first_time = timestamp
            last_time = timestamp

            if msg == 'explicit deny':
                user_agent = entry.get('user_agent', 'Unknown')
                path = entry.get('path', 'Unknown')
                check_result = entry.get('check_result', {})
                rule_name = check_result.get('name', 'Unknown')

                blocked_bots[user_agent] += 1
                blocked_pages[path] += 1

                if user_agent not in bot_block_techniques:
                    bot_block_techniques[user_agent] = set()
                bot_block_techniques[user_agent].add(rule_name)

    end_time = time.time()
    current, peak = tracemalloc.get_traced_memory()
    tracemalloc.stop()

    return {
        'first_time': first_time,
        'last_time': last_time,
        'blocked_bots': blocked_bots.most_common(),
        'blocked_pages': blocked_pages.most_common(10),
        'bot_block_techniques': bot_block_techniques,
        'line_count': line_count,
        'parse_time': end_time - start_time,
        'peak_memory': peak,
    }

def format_datetime(ts):
    if ts:
        dt = datetime.fromisoformat(ts.replace('Z', '+00:00'))
        return dt.strftime('%Y-%m-%d %H:%M:%S')
    return 'N/A'

def main():
    if len(sys.argv) < 2:
        print(f"Usage: {sys.argv[0]} <log_file>")
        sys.exit(1)

    filepath = sys.argv[1]

    if not os.path.exists(filepath):
        print(f"Error: File '{filepath}' not found")
        sys.exit(1)

    file_size = os.path.getsize(filepath)

    results = parse_anubis_log(filepath)

    print("=" * 60)
    print("📊 ANUBIS LOG ANALYSIS")
    print("=" * 60)
    print(f"\n🕐 Log Time Range:")
    print(f"   First Log: {format_datetime(results['first_time'])}")
    print(f"   Last Log:  {format_datetime(results['last_time'])}")

    print(f"\n{'=' * 60}")
    print("🤖 BLOCKED BOTS (Top 20)")
    print("=" * 60)
    for bot, count in results['blocked_bots'][:20]:
        techniques = ', '.join(sorted(results['bot_block_techniques'].get(bot, ['Unknown'])))
        print(f"   {count:>6} - {bot}")
        print(f"            Technique: {techniques}")

    print(f"\n{'=' * 60}")
    print("🚫 TOP 10 BLOCKED PAGES")
    print("=" * 60)
    for page, count in results['blocked_pages']:
        print(f"   {count:>6} - {page}")

    print(f"\n{'=' * 60}")
    print("⚙️  PARSER STATISTICS")
    print("=" * 60)
    lps = results['line_count'] / results['parse_time'] if results['parse_time'] > 0 else 0
    print(f"   📄 Lines processed:    {results['line_count']:,}")
    print(f"   💾 File size:         {format_bytes(file_size)}")
    print(f"   ⚡ Lines per second:  {lps:,.2f}")
    print(f"   🧠 Peak memory used:  {format_bytes(results['peak_memory'])}")

if __name__ == '__main__':
    main()

sample output
`============================================================
📊 ANUBIS LOG ANALYSIS

🕐 Log Time Range:
First Log: 2026-05-31 11:36:37
Last Log: 2026-05-31 13:25:50

============================================================
🤖 BLOCKED BOTS (Top 20)

12463 - Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai.com/gptbot)
        Technique: bot/block-ai-scrapers
 8188 - Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36
        Technique: bot/block-ai-scrapers

============================================================
🚫 TOP 10 BLOCKED PAGES

20633 - /catalogo/Search/Results
    2 - /catalogo/MyResearch/SaveSearch
    1 - /catalogo/Record/UMA-19027/Details
    1 - /catalogo/Record/UMA-1413
    1 - /catalogo/Record/UMA-4044/Description
    1 - /catalogo/Record/UMA-7606/Description
    1 - /catalogo/Record/UMA-6485/Description
    1 - /catalogo/Record/UMA-15929/Details
    1 - /catalogo/Record/UMA-2883/Details
    1 - /catalogo/Record/UMA-14301/Description

============================================================
⚙ PARSER STATISTICS

📄 Lines processed: 21,843
💾 File size: 13.08 MB
⚡ Lines per second: 14,515.47
🧠 Peak memory used: 27.21 KB`