Count Julia Discourse [ANN] posts over time and plot the trend (AlgebraOfGraphics/CairoMakie)

ann_over_time.jl

using Pkg
Pkg.activate(; temp = true)
Pkg.add(["HTTP", "JSON3", "DataFrames", "CairoMakie", "AlgebraOfGraphics"])

using HTTP, JSON3, DataFrames, Dates, CairoMakie, AlgebraOfGraphics
const AoG = AlgebraOfGraphics

const SEARCH_URL = "https://discourse.julialang.org/search.json"

function search_page(query, page)
    for attempt in 1:6
        resp = HTTP.get(SEARCH_URL; query = ["q" => query, "page" => string(page)],
                        status_exception = false)
        resp.status == 200 && return JSON3.read(resp.body)
        resp.status == 429 ? sleep(5 * attempt) : error("HTTP $(resp.status)")
    end
    error("too many retries")
end

parse_date(s) = Date(DateTime(s[1:19], dateformat"yyyy-mm-ddTHH:MM:SS"))

function topics_in_month(year, month)
    start = Date(year, month)
    query = "ANN in:title after:$(start) before:$(start + Month(1))"
    found = Dict{Int,Tuple{String,Date}}()
    page = 1
    while true
        result = search_page(query, page)
        isempty(result.topics) && break
        for t in result.topics
            found[t.id] = (String(t.title), parse_date(String(t.created_at)))
        end
        result.grouped_search_result.more_full_page_results in (nothing, false) && break
        page += 1
        sleep(0.3)
    end
    found
end

function fetch_ann_topics(; from = Date(2016, 11), to = today())
    rows = NamedTuple[]
    for m in from:Month(1):to
        for (id, (title, date)) in topics_in_month(year(m), month(m))
            push!(rows, (; id, title, date))
        end
        sleep(0.2)
    end
    unique(DataFrame(rows), :id)
end

df = fetch_ann_topics()

is_ann_tag(title) = occursin(r"^\s*(\[\s*)?ann\b"i, title) && !occursin("Ann Arbor", title)
filter!(:title => is_ann_tag, df)

df.month = [Date(year(d), month(d)) for d in df.date]
counts = combine(groupby(df, :month), nrow => :n)
allmonths = minimum(df.month):Month(1):maximum(df.month)
bymonth = Dict(counts.month .=> counts.n)
monthly = DataFrame(date = collect(allmonths), n = [get(bymonth, m, 0) for m in allmonths])

df.year = year.(df.date)
yearly = combine(groupby(df, :year), nrow => :n)
sort!(yearly, :year)
last_date = maximum(df.date)
current_year = year(last_date)
elapsed_days = Dates.value(last_date - Date(current_year)) + 1
yearly.annualized = [r.year == current_year ? round(Int, r.n * 365 / elapsed_days) : r.n
                     for r in eachrow(yearly)]
yearly.partial = yearly.year .== current_year

era_start = Date(2025, 2)
era = DataFrame(lo = [era_start], hi = [maximum(monthly.date)],
                x = [era_start - Month(1)], y = [maximum(monthly.n) + 1.0])

fig = Figure(size = (1000, 760))

era_layers = AoG.data(era) * (
    AoG.mapping(:lo, :hi) * AoG.visual(VSpan, color = (:orange, 0.1)) +
    AoG.mapping(:lo) * AoG.visual(VLines, color = (:orange, 0.6), linestyle = :dash) +
    AoG.mapping(:x, :y) * AoG.visual(Makie.Text, text = "Claude Code / coding-agent era",
        color = :darkorange, align = (:right, :top), fontsize = 12))
# degree = 1: AoG's default degree-2 loess collapses on Date-typed x, because dates convert
# to a ~1e11 numeric scale that ill-conditions the quadratic term; a local-linear fit is robust.
monthly_layers = AoG.data(monthly) * AoG.mapping(:date => "", :n) * (
    AoG.visual(Scatter, color = (:steelblue, 0.4), markersize = 7) +
    AoG.smooth(span = 0.4, degree = 1) * AoG.visual(color = :firebrick))
AoG.draw!(fig[1, 1], era_layers + monthly_layers;
          axis = (; title = "Julia Discourse [ANN] posts per month", ylabel = "posts / month"))

bars = AoG.data(filter(:partial => identity, yearly)) * AoG.mapping(:year, :annualized) *
           AoG.visual(BarPlot, color = (:firebrick, 0.2)) +
       AoG.data(yearly) * AoG.mapping(:year, :n, color = :partial) * AoG.visual(BarPlot)
grid = AoG.draw!(fig[2, 1], bars,
                 AoG.scales(Color = (; palette = [:steelblue, :firebrick], legend = false));
                 axis = (; title = "[ANN] posts per year ($(current_year) annualized from $(elapsed_days) days)",
                         xlabel = "year", ylabel = "posts / year",
                         xticks = minimum(yearly.year):maximum(yearly.year)))
for r in eachrow(yearly)
    label = r.partial ? "$(r.n)\n(→$(r.annualized))" : string(r.n)
    text!(grid[1].axis, r.year, r.partial ? r.annualized : r.n;
          text = label, align = (:center, :bottom), fontsize = 11)
end

save("julia_ann_posts.png", fig)
fig