Cohen Kappa Score

compute_kappa.py

import numpy as np
import pandas as pd
from sklearn.metrics import cohen_kappa_score, accuracy_score


A_T1 = [5,5,3,5,4,5,5,4,4,5]
B_T1 = [5,4,5,4,4,4,3,5,4,3]

A_T2 = [5,5,3,5,5,5,5,5,5,2]
B_T2 = [2,4,3,3,4,3,4,2,1,1]

A_T3 = [5,5,5,5,5,5,4,5,5,3]
B_T3 = [3,3,4,3,4,3,4,3,3,2]

A_OV = [3,3,5,4,5,4,3,3]
B_OV = [4,4,3,2,4,4,1,3]

A_all = A_T1 + A_T2 + A_T3 + A_OV
B_all = B_T1 + B_T2 + B_T3 + B_OV

blocks = {
    "All":     (A_all, B_all),
    "T1":      (A_T1, B_T1),
    "T2":      (A_T2, B_T2),
    "T3":      (A_T3, B_T3),
    "Overall": (A_OV,  B_OV),
}

def summarize(a, b):
    a = np.asarray(a); b = np.asarray(b)
    return {
        "n": len(a),
        "agree_pct": 100 * accuracy_score(a, b),
        "kappa_unw": cohen_kappa_score(a, b, weights=None),
        "kappa_w_lin": cohen_kappa_score(a, b, weights="linear"),
        "kappa_w_quad": cohen_kappa_score(a, b, weights="quadratic"),
        "mean_A": float(a.mean()),
        "mean_B": float(b.mean()),
        "mean_diff_AminusB": float(a.mean() - b.mean()),
    }

rows = []
for name, (a, b) in blocks.items():
    d = summarize(a, b)
    d["block"] = name
    rows.append(d)

df = pd.DataFrame(rows).set_index("block")
with pd.option_context("display.float_format", "{:.3f}".format):
    print(df[["n", "agree_pct", "kappa_unw", "kappa_w_lin", "kappa_w_quad",
              "mean_A", "mean_B", "mean_diff_AminusB"]])