entro.py - simple entropy check to evaluate whether compression is worth it

entro.py

from collections import Counter
from typing import Union


def should_compress(
        data: Union[bytes | bytearray | memoryview],
        /, *,
        threshold: int = 128) -> bool:
    # do not compress if buffer size is < threshold
    if len(data) < threshold:
        return False

    # count unique bytes
    frequency = Counter(data)

    # compress if less than 90% unique bytes
    return (float(len(frequency)) / float(min(len(data), 256))) < 0.9