Persian Newspapers Figures

visualize_findings.py

import sqlite3
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from matplotlib.colors import to_rgb
from matplotlib.ticker import MaxNLocator, PercentFormatter, ScalarFormatter


DB_PATH = Path("dh_stats.sqlite")
OUT_DIR = Path("dh_stats_out/figures")
CAPTIONS_PATH = OUT_DIR / "figures.md"

FIGURES = []

MIN_PAGES_PER_YEAR = 50
MIN_PAGES_PER_JOURNAL_SLICE = 1000
MIN_LINES_FOR_TEXT_ZONE = 50
MIN_YEARS_PER_JOURNAL = 8
MIN_CHARS_FOR_PAGE = 100

SKEW_THRESHOLD = 0.5
SKEW_BAND_EDGES = [-0.001, 0.0001, 0.25, 0.5, 1.0, 2.0, 90.0]
SKEW_BAND_LABELS = ["exactly 0\u00b0", "0\u20130.25\u00b0", "0.25\u20130.5\u00b0", "0.5\u20131\u00b0", "1\u20132\u00b0", "over 2\u00b0"]
SKEW_BAND_COLORS = ["#d1d5db", "#c6dbef", "#9ecae1", "#6baed6", "#3182bd", "#08519c"]

CONDITION_METRICS = [
    {
        "column": "megapixels",
        "title": "Scan quality",
        "number": 3,
        "legend": "Horizontal axis: publication year. Vertical axis: megapixels per scanned page, logarithmic. Line: yearly median. "
        "Shaded band: first to third quartile.\n\n"
        f"Years holding fewer than {MIN_PAGES_PER_YEAR} pages are omitted as too thin to average. The quantity is measured on the source "
        "image file itself.",
        "journal_title": "Scan quality per newspaper",
        "journal_number": 4,
        "journal_legend": "Horizontal axis: publication year. Vertical axis: megapixels per scanned page, logarithmic. One coloured line "
        "per newspaper, each the yearly median across that title's pages.\n\n"
        f"Only titles holding at least {MIN_YEARS_PER_JOURNAL} years of {MIN_PAGES_PER_YEAR} pages or more are drawn, which selects the "
        "five largest. The quantity is measured on the image file itself.",
        "ylabel": "Megapixels",
        "log": True,
        "kind": "median",
    },
    {
        "column": "crooked",
        "title": "Skewed scans",
        "number": 5,
        "legend": "Horizontal axis: publication year. Vertical axis: share of scans skewed more than "
        f"{SKEW_THRESHOLD} degrees. One point per year.",
        "journal_title": None,
        "journal_number": None,
        "journal_legend": None,
        "ylabel": f"Share skewed over {SKEW_THRESHOLD}\u00b0",
        "log": False,
        "kind": "share",
    },
]
ITALAAT_FIRST_YEAR = 1943
EXCLUDED_JOURNAL_YEARS = {("Italaat", 1942)}

FIRST_YEAR = 1870
LAST_YEAR = 1970

AREA_COLS = ["area_text", "area_graphic", "area_table", "area_margin", "area_running_title", "area_numbering", "area_other"]

JOURNAL_LABELS = {"Italaat": "Ettela'at", "Ah̲tar": "Akhtar", "Iran": "Irān"}

BLUE = "#31688e"
TEAL = "#21918c"
GREEN = "#35b779"
GOLD = "#f2a900"
RED = "#c44e52"
GRAY = "#6b7280"
LIGHT_GRAY = "#d1d5db"

JOURNAL_COLORS = [BLUE, GOLD, GREEN, RED, TEAL]

JOURNAL_SLICE_COLORS = [
    "#2a78d6",
    "#eb6834",
    "#0d9488",
    "#e34948",
    "#eda100",
    "#4a3aa7",
    "#65a30d",
    "#a855f7",
    "#1baf7a",
    "#a16207",
    "#0ea5e9",
    "#e87ba4",
    "#008300",
]

LAYOUT_COUNTS = [
    {"column": "n_lines", "title": "Text lines", "ylabel": "Lines per page", "color": BLUE},
    {"column": "n_columns", "title": "Text columns", "ylabel": "Columns per page", "color": GRAY},
    {"column": "n_boxes", "title": "Layout boxes", "ylabel": "Boxes per page", "color": TEAL},
]


def _style():
    plt.rcParams.update(
        {
            "figure.dpi": 150,
            "savefig.dpi": 180,
            "savefig.bbox": "tight",
            "font.size": 10,
            "axes.titlesize": 13,
            "axes.titleweight": "bold",
            "axes.labelsize": 10,
            "axes.spines.top": False,
            "axes.spines.right": False,
            "axes.grid": True,
            "axes.grid.axis": "y",
            "grid.alpha": 0.2,
            "legend.frameon": False,
        }
    )


def _drop_segmentation_failures(data):
    failed = (data["area_text"] == 0) & (data["n_lines"] > MIN_LINES_FOR_TEXT_ZONE)
    if failed.any():
        print(f"Dropped {int(failed.sum())} pages whose text lines all sit outside a text zone")
    return data[~failed]


def _annual_mean(data, column, label):
    grouped = data.groupby("greg_year")[column]
    stats = pd.DataFrame({"mean": grouped.mean(), "sem": grouped.sem(), "pages": grouped.size()})
    thin = stats["pages"] < MIN_PAGES_PER_YEAR
    if thin.any():
        print(f"{label}: dropped {int(thin.sum())} years holding fewer than {MIN_PAGES_PER_YEAR} pages")
    stats = stats[~thin]
    if stats.empty:
        return stats
    return stats.reindex(range(int(stats.index.min()), int(stats.index.max()) + 1))


def _graphic_share(pages):
    data = pages[pages[AREA_COLS].notna().all(axis=1) & pages["page_w"].notna() & pages["page_h"].notna()].copy()
    data["greg_year"] = data["greg_year"].astype(int)
    data = _drop_segmentation_failures(data)
    page_area = data["page_w"] * data["page_h"]
    data = data[page_area > 0].copy()
    data["graphic_share"] = data["area_graphic"] / (data["page_w"] * data["page_h"])
    return data


def _drop_excluded_journal_years(data):
    keys = pd.Series(list(zip(data["journal"], data["greg_year"])), index=data.index)
    keep = ~keys.isin(EXCLUDED_JOURNAL_YEARS)
    if (~keep).any():
        print(f"Dropped {int((~keep).sum())} pages from journal-years held in an incomparable scanning batch")
    return data[keep]


def _drop_dead_pages(data):
    dead = data["n_chars"].fillna(0) < MIN_CHARS_FOR_PAGE
    if dead.any():
        print(f"Dropped {int(dead.sum())} pages that yielded no recognised text")
    return data[~dead]


def _weighted(group, value, weight="n_words"):
    valid = group[value].notna() & group[weight].notna()
    if not valid.any() or group.loc[valid, weight].sum() == 0:
        return np.nan
    return np.average(group.loc[valid, value], weights=group.loc[valid, weight])


def _annual(group):
    rows = []
    for year, data in group.groupby("greg_year"):
        rows.append(
            {
                "year": int(year),
                "pages": len(data),
                "confidence": _weighted(data, "wc_mean"),
                "low_confidence": _weighted(data, "wc_frac_lt50"),
                "columns": data["n_columns"].mean(),
                "boxes": data["n_boxes"].mean(),
                "line_chars": data["mean_line_chars"].mean(),
                "page_height": data["page_h"].median(),
                "density": data["chars_per_mpx"].mean(),
            }
        )
    return pd.DataFrame(rows).set_index("year").sort_index()


def _save(fig, number, title, legend):
    name = f"figure_{number:02d}.png"
    fig.savefig(OUT_DIR / name, facecolor="white")
    plt.close(fig)
    FIGURES.append({"number": number, "name": name, "title": title, "legend": legend})


def _write_captions():
    lines = []
    for figure in sorted(FIGURES, key=lambda figure: figure["number"]):
        caption = f"Figure {figure['number']}. {figure['title']}"
        lines.append(caption)
        lines.append("")
        lines.append(figure["legend"])
        lines.append("")
        lines.append(f"![{caption}]({figure['name']})")
        lines.append("")
    CAPTIONS_PATH.write_text("\n".join(lines))


def _skew_frame(pages):
    data = pages[pages["deskew_deg"].notna()].copy()
    data["greg_year"] = data["greg_year"].astype(int)
    data["tilt"] = data["deskew_deg"].abs()
    data["band"] = pd.cut(data["tilt"], SKEW_BAND_EDGES, labels=False)
    return data


def _big_journals(pages):
    counts = pages.groupby(["journal", "greg_year"]).size()
    thick = counts[counts >= MIN_PAGES_PER_YEAR].groupby("journal").size()
    eligible = thick[thick >= MIN_YEARS_PER_JOURNAL].index
    ranked = pages[pages["journal"].isin(eligible)].groupby("journal").size().sort_values(ascending=False)
    return list(ranked.index)


def _band(ax, stats, color, label, linestyle="-"):
    ax.plot(stats.index, stats["mean"], color=color, marker="o", markersize=3, linestyle=linestyle, label=label)
    ax.fill_between(
        stats.index,
        stats["mean"] - 2 * stats["sem"],
        stats["mean"] + 2 * stats["sem"],
        color=color,
        alpha=0.18,
        linewidth=0,
    )


def plot_italaat_layout(pages):
    data = pages[(pages["journal"] == "Italaat") & pages["greg_year"].between(ITALAAT_FIRST_YEAR, LAST_YEAR)]
    annual = _annual(data).reindex(range(ITALAAT_FIRST_YEAR, LAST_YEAR + 1))

    fig, axes = plt.subplots(2, 1, figsize=(10, 7), sharex=True)

    axes[0].plot(annual.index, annual["confidence"], color=BLUE, marker="o", label="Mean OCR confidence")
    axes[0].plot(annual.index, annual["low_confidence"], color=RED, marker="o", label="Share below 0.5")
    axes[0].set_ylim(0, 1)
    axes[0].set_ylabel("Share / confidence")
    axes[0].set_xlabel("Publication year")
    axes[0].tick_params(axis="x", labelbottom=True)
    axes[0].legend(ncol=2, loc="lower left")

    axes[1].plot(annual.index, annual["boxes"], color=GOLD, marker="o", label="Layout boxes")
    axes[1].plot(annual.index, annual["line_chars"], color=TEAL, marker="o", label="Characters per line")
    axes[1].set_ylabel("Mean per page/line")
    axes[1].set_xlabel("Publication year")
    axes[1].legend(ncol=2, loc="upper left")

    _save(
        fig,
        26,
        f"Ettela'at: layout and recognition quality, {ITALAAT_FIRST_YEAR}-70",
        "Both panels: horizontal axis publication year. Upper vertical axis: mean OCR confidence and share of words below 0.5, both "
        "weighted by words per page. Lower: mean layout boxes per page and mean characters per line.",
    )


def plot_italaat_layout_metrics(pages):
    data = _drop_dead_pages(_drop_excluded_journal_years(pages[(pages["journal"] == "Italaat") & pages["greg_year"].between(ITALAAT_FIRST_YEAR, LAST_YEAR)])).copy()
    data["greg_year"] = data["greg_year"].astype(int)
    boxes = _annual_mean(data, "n_boxes", "layout boxes")
    line_chars = _annual_mean(data, "mean_line_chars", "characters per line")

    fig, ax = plt.subplots(figsize=(11, 5))
    _band(ax, boxes, GOLD, "Mean layout boxes per page")
    ax.set_ylabel("Layout boxes per page", color=GOLD)
    ax.set_xlabel("Publication year")

    ax2 = ax.twinx()
    _band(ax2, line_chars, TEAL, "Mean characters per line")
    ax2.set_ylabel("Characters per line", color=TEAL)
    ax2.grid(False)

    lines = ax.get_lines() + ax2.get_lines()
    ax.legend(lines, [line.get_label() for line in lines], loc="upper left")
    _save(
        fig,
        27,
        f"Ettela'at: layout boxes and characters per line, {ITALAAT_FIRST_YEAR}-70",
        "Horizontal axis: publication year. Left axis, gold: mean layout boxes per page. Right axis, teal: mean characters per "
        "recognised line. Shaded bands: two standard errors.",
    )


def plot_pages_per_year(all_pages):
    data = all_pages[all_pages["greg_year"].notna()].copy()
    data["greg_year"] = data["greg_year"].astype(int)
    data = data.drop_duplicates(["journal", "volume", "issue", "page"])
    data["page_count"] = np.where(data["two_page"] == 1, 2, 1)
    counts = data.groupby("greg_year")["page_count"].sum()
    counts = counts.reindex(range(int(counts.index.min()), int(counts.index.max()) + 1), fill_value=0)

    fig, ax = plt.subplots(figsize=(11, 5))
    ax.bar(counts.index, counts.values, color=GOLD, width=0.8)
    ax.set_ylabel("Distinct pages (double-page scans count as two)")
    ax.set_xlabel("Publication year")
    ax.xaxis.set_major_locator(MaxNLocator(integer=True, nbins=12))
    _save(
        fig,
        2,
        "Distinct pages per year",
        "Horizontal axis: publication year. Vertical axis: estimated distinct pages held for that year. One bar per year.",
    )


def plot_pages_by_journal(all_pages):
    data = all_pages[all_pages["greg_year"].notna()].copy()
    data = data.drop_duplicates(["journal", "volume", "issue", "page"])
    data["page_count"] = np.where(data["two_page"] == 1, 2, 1)
    counts = data.groupby("journal")["page_count"].sum().sort_values(ascending=False)
    small = counts[counts < MIN_PAGES_PER_JOURNAL_SLICE]
    counts = counts[counts >= MIN_PAGES_PER_JOURNAL_SLICE]
    counts.index = [JOURNAL_LABELS.get(name, name) for name in counts.index]
    colors = list(JOURNAL_SLICE_COLORS[: len(counts)])
    if not small.empty:
        print(f"Folded {len(small)} titles holding fewer than {MIN_PAGES_PER_JOURNAL_SLICE} pages into one slice")
        counts["Others"] = small.sum()
        colors.append(LIGHT_GRAY)
    total = counts.sum()

    fig, ax = plt.subplots(figsize=(9.5, 6))
    wedges, _, percents = ax.pie(
        counts.values,
        colors=colors,
        startangle=90,
        counterclock=False,
        autopct=lambda share: f"{share:.1f}%" if share >= 4 else "",
        pctdistance=0.74,
        wedgeprops={"edgecolor": "white", "linewidth": 1.2},
        textprops={"fontsize": 9, "fontweight": "bold"},
    )
    for text, color in zip(percents, colors):
        red, green, blue = to_rgb(color)
        text.set_color("white" if 0.299 * red + 0.587 * green + 0.114 * blue < 0.6 else "#1f2937")
    ax.set_axis_off()
    ax.legend(
        wedges,
        [f"{name} - {count:,} ({count / total:.1%})" for name, count in counts.items()],
        loc="center left",
        bbox_to_anchor=(0.98, 0.5),
        fontsize=9,
    )
    _save(
        fig,
        1,
        "Pages distribution by newspaper",
        "One coloured slice per newspaper, sized by the distinct estimated pages held for it and ordered clockwise from largest. Titles "
        f"holding fewer than {MIN_PAGES_PER_JOURNAL_SLICE:,} pages are folded into one grey slice titled \u201cOthers\u201d. The legend "
        "gives the page count and share of every slice.",
    )


def plot_graphic_real_estate(pages):
    data = _graphic_share(pages)
    annual = _annual_mean(data, "graphic_share", "graphic share")
    observed = annual.dropna(subset=["mean"])

    fig, ax = plt.subplots(figsize=(11, 5))
    _band(ax, annual, GREEN, "Mean share per page")
    ax.scatter(
        observed.index,
        observed["mean"],
        s=np.sqrt(observed["pages"]) * 1.6,
        color=GREEN,
        edgecolor="white",
        linewidth=0.5,
        zorder=3,
    )
    ax.yaxis.set_major_formatter(PercentFormatter(1))
    ax.set_ylim(bottom=0)
    ax.set_ylabel("Share of scanned page area")
    ax.set_xlabel("Publication year")
    ax.legend(["Mean share per page", "\u00b12 standard errors"], loc="upper left")
    _save(
        fig,
        20,
        "Graphic content over time",
        "Horizontal axis: publication year. Vertical axis: share of the scanned page given to graphic content. Point: yearly mean across "
        "pages, with area scaling to the number of pages. Shaded band: two standard errors.",
    )


def plot_graphic_real_estate_by_journal(pages):
    data = _drop_excluded_journal_years(_graphic_share(pages))
    journals = _big_journals(data)

    fig, ax = plt.subplots(figsize=(11, 5.5))
    for color, journal in zip(JOURNAL_COLORS, journals):
        stats = _annual_mean(data[data["journal"] == journal], "graphic_share", journal)
        _band(ax, stats, color, JOURNAL_LABELS.get(journal, journal))
    ax.yaxis.set_major_formatter(PercentFormatter(1))
    ax.set_ylim(bottom=0)
    ax.set_ylabel("Share of scanned page area")
    ax.set_xlabel("Publication year")
    ax.legend(ncol=3, loc="upper left")
    _save(
        fig,
        21,
        "Graphic content over time by newspaper",
        "Horizontal axis: publication year. Vertical axis: share of the scanned page given to graphic content. One coloured line per "
        "newspaper, with a band of two standard errors.",
    )


def _source_condition(pages):
    data = pages[pages["page_w"].notna() & pages["page_h"].notna()].copy()
    data["greg_year"] = data["greg_year"].astype(int)
    page_area = data["page_w"] * data["page_h"]
    data["megapixels"] = page_area / 1e6
    data["skew"] = data["deskew_deg"].abs()
    data["crooked"] = (data["skew"] > SKEW_THRESHOLD).astype(float)
    data["coverage"] = data[AREA_COLS].sum(axis=1) / page_area
    return data


def _annual_spread(data, column, kind):
    grouped = data[data[column].notna()].groupby("greg_year")[column]
    if kind == "share":
        stats = pd.DataFrame({"mid": grouped.mean(), "low": np.nan, "high": np.nan, "pages": grouped.size()})
    else:
        mid = grouped.mean() if kind == "mean" else grouped.median()
        stats = pd.DataFrame({"mid": mid, "low": grouped.quantile(0.25), "high": grouped.quantile(0.75), "pages": grouped.size()})
    stats = stats[stats["pages"] >= MIN_PAGES_PER_YEAR]
    if stats.empty:
        return stats
    return stats.reindex(range(int(stats.index.min()), int(stats.index.max()) + 1))


def plot_condition_metric(pages, metric):
    stats = _annual_spread(_source_condition(pages), metric["column"], metric["kind"])
    share = metric["kind"] == "share"

    fig, ax = plt.subplots(figsize=(11, 5))
    if share:
        ax.yaxis.set_major_formatter(PercentFormatter(1))
        ax.set_ylim(0, 1)
    else:
        ax.fill_between(stats.index, stats["low"], stats["high"], color=BLUE, alpha=0.18, linewidth=0)
    ax.plot(stats.index, stats["mid"], color=BLUE, marker="o", markersize=3)
    if metric["log"]:
        ax.set_yscale("log")
        ax.set_yticks([2, 3, 5, 10, 30, 100])
        ax.yaxis.set_major_formatter(ScalarFormatter())
        ax.minorticks_off()
    ax.set_ylabel(metric["ylabel"])
    ax.set_xlabel("Publication year")
    _save(
        fig,
        metric["number"],
        metric["title"],
        metric["legend"],
    )


def plot_condition_metric_by_journal(pages, metric):
    data = _drop_excluded_journal_years(_source_condition(pages))
    journals = _big_journals(data)
    share = metric["kind"] == "share"

    fig, ax = plt.subplots(figsize=(11, 5.5))
    for color, journal in zip(JOURNAL_COLORS, journals):
        stats = _annual_spread(data[data["journal"] == journal], metric["column"], metric["kind"])
        ax.plot(stats.index, stats["mid"], color=color, marker="o", markersize=3, label=JOURNAL_LABELS.get(journal, journal))
    if share:
        ax.yaxis.set_major_formatter(PercentFormatter(1))
        ax.set_ylim(0, 1)
    if metric["log"]:
        ax.set_yscale("log")
        ax.set_yticks([2, 3, 5, 10, 30, 100])
        ax.yaxis.set_major_formatter(ScalarFormatter())
        ax.minorticks_off()
    ax.set_ylabel(metric["ylabel"])
    ax.set_xlabel("Publication year")
    ax.legend(ncol=3, loc="upper left")
    _save(
        fig,
        metric["journal_number"],
        metric["journal_title"],
        metric["journal_legend"],
    )


def _layout_counts(pages):
    data = pages[pages["n_boxes"].notna() & pages["n_columns"].notna() & pages["n_lines"].notna()].copy()
    data["greg_year"] = data["greg_year"].astype(int)
    return _drop_segmentation_failures(data)


def plot_layout_counts(pages):
    data = _layout_counts(pages)

    fig, axes = plt.subplots(len(LAYOUT_COUNTS), 1, figsize=(11, 9.5), sharex=True)
    for ax, metric in zip(axes, LAYOUT_COUNTS):
        stats = _annual_spread(data, metric["column"], "mean")
        ax.fill_between(stats.index, stats["low"], stats["high"], color=metric["color"], alpha=0.18, linewidth=0)
        ax.plot(stats.index, stats["mid"], color=metric["color"], marker="o", markersize=3)
        ax.set_ylabel(metric["ylabel"])
        ax.set_ylim(bottom=0)
        ax.set_title(metric["title"], loc="left", fontsize=11)
    axes[-1].set_xlabel("Publication year")
    fig.align_ylabels(axes)
    _save(
        fig,
        14,
        "Page layout over time",
        "Horizontal axis: publication year, shared by all three panels. Vertical axes, from top to bottom: text lines, text columns and "
        "layout boxes per page. Line: yearly mean. Shaded band: first to third quartile.",
    )


def plot_layout_counts_by_journal(pages):
    data = _drop_excluded_journal_years(_layout_counts(pages))
    journals = _big_journals(data)

    fig, axes = plt.subplots(len(LAYOUT_COUNTS), 1, figsize=(11, 10), sharex=True)
    for ax, metric in zip(axes, LAYOUT_COUNTS):
        for color, journal in zip(JOURNAL_COLORS, journals):
            stats = _annual_spread(data[data["journal"] == journal], metric["column"], "mean")
            ax.plot(stats.index, stats["mid"], color=color, marker="o", markersize=3, label=JOURNAL_LABELS.get(journal, journal))
        ax.set_ylabel(metric["ylabel"])
        ax.set_ylim(bottom=0)
        ax.set_title(metric["title"], loc="left", fontsize=11)
    axes[0].legend(ncol=3, loc="upper left")
    axes[-1].set_xlabel("Publication year")
    fig.align_ylabels(axes)
    _save(
        fig,
        15,
        "Page layout over time by newspaper",
        "Horizontal axis: publication year, shared by all three panels. Vertical axes, from top to bottom: text lines, text columns and "
        "layout boxes per page. One coloured line per newspaper in every panel, each the yearly mean across that title's pages.",
    )

def plot_ocr_quality(pages):
    data = _drop_dead_pages(pages[pages["wc_mean"].notna() & pages["wc_frac_lt50"].notna()]).copy()
    data["greg_year"] = data["greg_year"].astype(int)
    confidence = _annual_mean(data, "wc_mean", "OCR confidence")
    failures = _annual_mean(data, "wc_frac_lt50", "words below 0.5")

    fig, axes = plt.subplots(2, 1, figsize=(11, 6.5), sharex=True, gridspec_kw={"height_ratios": [2, 1]})
    _band(axes[0], confidence, BLUE, "Mean OCR confidence")
    axes[0].set_ylabel("Mean OCR confidence (0-1)")
    axes[0].legend(loc="lower left")

    _band(axes[1], failures, RED, "Share of words below 0.5")
    axes[1].yaxis.set_major_formatter(PercentFormatter(1))
    axes[1].set_ylim(bottom=0)
    axes[1].set_ylabel("Share of words below 0.5")
    axes[1].set_xlabel("Publication year")
    axes[1].legend(loc="upper left")
    _save(
        fig,
        23,
        "Recognition quality over time",
        "Horizontal axis, both panels: publication year. Upper vertical axis: mean OCR confidence per page on a 0-1 scale. Lower: "
        "share of words whose confidence falls below 0.5. Shaded bands: two standard errors.",
    )


def plot_ocr_quality_by_journal(pages):
    data = _drop_dead_pages(pages[pages["wc_mean"].notna()]).copy()
    data["greg_year"] = data["greg_year"].astype(int)
    data = _drop_excluded_journal_years(data)
    journals = _big_journals(data)

    fig, ax = plt.subplots(figsize=(11, 5.5))
    for color, journal in zip(JOURNAL_COLORS, journals):
        stats = _annual_mean(data[data["journal"] == journal], "wc_mean", journal)
        _band(ax, stats, color, JOURNAL_LABELS.get(journal, journal))
    ax.set_ylabel("Mean OCR confidence")
    ax.set_xlabel("Publication year")
    ax.legend(ncol=3, loc="lower left")
    _save(
        fig,
        24,
        "Recognition quality within each newspaper",
        "Horizontal axis: publication year. Vertical axis: mean OCR confidence per page. One colored line per newspaper, with a band of "
        "two standard errors.",
    )


def plot_skew_composition(pages):
    data = _drop_excluded_journal_years(_skew_frame(pages))
    journals = _big_journals(data)
    rows = [JOURNAL_LABELS.get(journal, journal) for journal in journals]
    positions = np.arange(len(journals))[::-1]

    fig, ax = plt.subplots(figsize=(11, 4.8))
    left = np.zeros(len(journals))
    for band, (label, color) in enumerate(zip(SKEW_BAND_LABELS, SKEW_BAND_COLORS)):
        widths = np.array([(data[data["journal"] == journal]["band"] == band).mean() for journal in journals])
        ax.barh(positions, widths, left=left, color=color, height=0.62, label=label, edgecolor="white", linewidth=0.6)
        left = left + widths
    for position, journal in zip(positions, journals):
        mean_tilt = data[data["journal"] == journal]["tilt"].mean()
        ax.text(1.01, position, f"mean {mean_tilt:.2f}\u00b0", va="center", fontsize=9, color=GRAY)
    ax.set_yticks(positions)
    ax.set_yticklabels(rows)
    ax.set_xlim(0, 1)
    ax.xaxis.set_major_formatter(PercentFormatter(1))
    ax.set_xlabel("Share of the title's scans")
    ax.grid(False)
    ax.legend(ncol=6, loc="upper center", bbox_to_anchor=(0.5, 1.13), fontsize=9)
    _save(
        fig,
        12,
        "Skewness across newspaper scans",
        "One bar per large newspaper, split into skew bands and running to 100 percent of that title's scans; the mean angle is printed "
        "at the right of each bar.",
    )


def plot_confidence_vs_columns(pages):
    data = _drop_excluded_journal_years(_drop_dead_pages(pages[pages["wc_mean"].notna() & pages["n_columns"].notna()]).copy())
    data["greg_year"] = data["greg_year"].astype(int)
    annual = data.groupby(["journal", "greg_year"]).agg(
        pages=("n_columns", "size"),
        columns=("n_columns", "mean"),
        confidence=("wc_mean", "mean"),
    )
    annual = annual[annual["pages"] >= MIN_PAGES_PER_YEAR].reset_index()
    journals = _big_journals(data)
    colors = dict(zip(journals, JOURNAL_COLORS))

    fig, ax = plt.subplots(figsize=(8, 5.5))
    others = annual[~annual["journal"].isin(journals)]
    ax.scatter(others["columns"], others["confidence"], s=np.sqrt(others["pages"]) * 2, color=LIGHT_GRAY, edgecolor="white", linewidth=0.5, label="Other titles")
    for journal in journals:
        subset = annual[annual["journal"] == journal]
        ax.scatter(subset["columns"], subset["confidence"], s=np.sqrt(subset["pages"]) * 2, color=colors[journal], edgecolor="white", linewidth=0.5, label=JOURNAL_LABELS.get(journal, journal))
    ax.set_ylim(0.65, 1.0)
    ax.set_xlabel("Detected text columns per page (mean)")
    ax.set_ylabel("Mean OCR confidence")
    ax.legend(ncol=3, loc="upper center", bbox_to_anchor=(0.5, -0.13), fontsize=8, frameon=True, edgecolor=LIGHT_GRAY, framealpha=1, borderpad=0.8)
    _save(
        fig,
        25,
        "Recognition quality against layout density",
        "One circle per title-year, placed by the mean number of text columns detected on its pages (horizontal) and its mean OCR "
        "confidence (vertical), with area scaling to pages behind it; the five largest titles coloured, the rest grey.",
    )


def plot_skew_effect(pages):
    data = _skew_frame(pages)
    data = _drop_dead_pages(data[data["wc_mean"].notna()]).copy()
    data["cohort"] = data["journal"] + "|" + data["greg_year"].astype(str)
    data["deviation"] = data["wc_mean"] - data.groupby("cohort")["wc_mean"].transform("mean")
    mixed = data[data.groupby("cohort")["tilt"].transform("std") > 0.01]
    raw = data.groupby("band")["wc_mean"].mean() - data["wc_mean"].mean()
    within = mixed.groupby("band")["deviation"].agg(["mean", "sem", "size"])
    positions = np.arange(len(SKEW_BAND_LABELS))[::-1]

    fig, ax = plt.subplots(figsize=(11, 5))
    ax.axvline(0, color=LIGHT_GRAY, linewidth=1)
    ax.hlines(positions, raw.reindex(range(len(SKEW_BAND_LABELS))), within["mean"].reindex(range(len(SKEW_BAND_LABELS))), color=LIGHT_GRAY, linewidth=2, zorder=1)
    ax.scatter(raw.reindex(range(len(SKEW_BAND_LABELS))), positions, color=RED, s=45, zorder=2, label="Raw, against the corpus average")
    ax.errorbar(
        within["mean"].reindex(range(len(SKEW_BAND_LABELS))),
        positions,
        xerr=2 * within["sem"].reindex(range(len(SKEW_BAND_LABELS))),
        fmt="o",
        color=TEAL,
        markersize=7,
        capsize=3,
        zorder=3,
        label="Within the same title and year",
    )
    labels = ax.get_yaxis_transform()
    for position, band in zip(positions, range(len(SKEW_BAND_LABELS))):
        ax.text(1.01, position, f"{int(within['size'].get(band, 0)):,} pages", transform=labels, va="center", fontsize=8, color=GRAY)
    ax.set_yticks(positions)
    ax.set_yticklabels(SKEW_BAND_LABELS)
    ax.set_ylim(-0.6, len(SKEW_BAND_LABELS) - 0.4)
    ax.margins(x=0.08)
    ax.set_xlabel("Difference in mean OCR confidence")
    ax.grid(axis="x", alpha=0.2)
    ax.legend(ncol=2, loc="lower center", bbox_to_anchor=(0.5, 1.02))
    _save(
        fig,
        13,
        "Skew effect on OCR confidence",
        "Horizontal axis: difference in mean OCR confidence. Vertical axis: skew band. Red: the band's average against the corpus "
        "average. Teal: the same comparison made only against other pages of the same title and year, with bars of two standard errors.",
    )


def plot_resolution_vs_confidence(pages):
    data = _drop_excluded_journal_years(_drop_dead_pages(pages[pages["page_w"].notna() & pages["page_h"].notna() & pages["wc_mean"].notna()]).copy())
    data["greg_year"] = data["greg_year"].astype(int)
    data["megapixels"] = data["page_w"] * data["page_h"] / 1e6
    annual = data.groupby(["journal", "greg_year"]).agg(
        pages=("megapixels", "size"),
        megapixels=("megapixels", "median"),
        confidence=("wc_mean", "mean"),
    )
    annual = annual[annual["pages"] >= MIN_PAGES_PER_YEAR].reset_index()
    journals = _big_journals(data)
    colors = dict(zip(journals, JOURNAL_COLORS))

    fig, ax = plt.subplots(figsize=(8, 5.5))
    others = annual[~annual["journal"].isin(journals)]
    ax.scatter(others["megapixels"], others["confidence"], s=np.sqrt(others["pages"]) * 2, color=LIGHT_GRAY, edgecolor="white", linewidth=0.5, label="Other titles")
    for journal in journals:
        subset = annual[annual["journal"] == journal]
        ax.scatter(subset["megapixels"], subset["confidence"], s=np.sqrt(subset["pages"]) * 2, color=colors[journal], edgecolor="white", linewidth=0.5, label=JOURNAL_LABELS.get(journal, journal))
    ax.set_xscale("log")
    ax.set_ylim(0.65, 1.0)
    ax.set_xlabel("Scanned page resolution (median megapixels, log scale)")
    ax.set_ylabel("Mean OCR confidence")
    ax.legend(ncol=3, loc="upper center", bbox_to_anchor=(0.5, -0.13), fontsize=8, frameon=True, edgecolor=LIGHT_GRAY, framealpha=1, borderpad=0.8)
    _save(
        fig,
        22,
        "Recognition quality against scan resolution",
        "One circle per title-year, placed by the median megapixels of its scanned pages (horizontal, logarithmic) and its mean OCR "
        "confidence (vertical), with area scaling to pages behind it; the five largest titles coloured, the rest grey.",
    )


def main():
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    _style()
    with sqlite3.connect(DB_PATH) as conn:
        all_pages = pd.read_sql_query("SELECT * FROM records", conn)
    all_pages = all_pages[all_pages["greg_year"].between(FIRST_YEAR, LAST_YEAR)].copy()
    pages = all_pages.copy()

    plot_pages_per_year(all_pages)
    plot_pages_by_journal(all_pages)
    for metric in CONDITION_METRICS:
        plot_condition_metric(pages, metric)
        if metric["journal_title"]:
            plot_condition_metric_by_journal(pages, metric)
        if metric["column"] == "crooked":
            plot_skew_composition(pages)
    plot_layout_counts(pages)
    plot_layout_counts_by_journal(pages)
    plot_ocr_quality(pages)
    plot_ocr_quality_by_journal(pages)
    plot_confidence_vs_columns(pages)
    plot_skew_effect(pages)
    plot_resolution_vs_confidence(pages)
    plot_italaat_layout(pages)
    plot_italaat_layout_metrics(pages)
    plot_graphic_real_estate(pages)
    plot_graphic_real_estate_by_journal(pages)
    _write_captions()
    print(f"Wrote {len(FIGURES)} figures and {CAPTIONS_PATH} to {OUT_DIR}")


if __name__ == "__main__":
    main()