Last active
April 12, 2026 04:09
-
-
Save chrimaho/701543f54f2c53ebc5fa9d47d8629b82 to your computer and use it in GitHub Desktop.
Matplotlib reference chart: common distribution shapes (normal, uniform, skew, bimodal, multimodal) + central tendency panel showing mean, median & mode on a right-skewed lognormal
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ── Imports ───────────────────────────────────────────────────────────────── | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| import matplotlib.gridspec as gridspec | |
| from scipy import stats | |
| # ── Colour palette ────────────────────────────────────────────────────────── | |
| C_BLUE = "#4C72B0" | |
| C_GREEN = "#55A868" | |
| C_ORANGE = "#DD8452" | |
| C_RED = "#C44E52" | |
| C_PURPLE = "#8172B2" | |
| C_TEAL = "#64B5CD" | |
| ALPHA = 0.35 | |
| def clean_ax(ax, title): | |
| """Remove clutter; keep only bottom spine.""" | |
| for spine in ["top", "right", "left"]: | |
| ax.spines[spine].set_visible(False) | |
| ax.spines["bottom"].set_color("#AAAAAA") | |
| ax.set_xticks([]) | |
| ax.set_yticks([]) | |
| ax.set_title(title, fontsize=12, fontweight="bold", pad=8) | |
| # ── Master figure (7 panels: 6 shape + 1 central-tendency) ──────────────── | |
| fig = plt.figure(figsize=(16, 14), facecolor="white") | |
| # Section label rows via a tall GridSpec | |
| outer = gridspec.GridSpec( | |
| 3, 1, | |
| figure=fig, | |
| height_ratios=[0.35, 4.5, 3.8], # [section-A label, shape grid, CT panel] | |
| hspace=0.12 | |
| ) | |
| # ── Section A header ──────────────────────────────────────────────────────── | |
| ax_head_A = fig.add_subplot(outer[0]) | |
| ax_head_A.axis("off") | |
| ax_head_A.text(0.5, 0.5, "1 | Data Shapes", | |
| ha="center", va="center", | |
| fontsize=15, fontweight="bold", color="#2c2c2c", | |
| transform=ax_head_A.transAxes) | |
| # ── 2 × 3 grid for the 6 shape panels ────────────────────────────────────── | |
| inner = gridspec.GridSpecFromSubplotSpec( | |
| 2, 3, | |
| subplot_spec=outer[1], | |
| hspace=0.55, wspace=0.35 | |
| ) | |
| axes_shape = [fig.add_subplot(inner[r, c]) for r in range(2) for c in range(3)] | |
| x = np.linspace(-4, 4, 400) | |
| # ── 1.1 Bell curve ───────────────────────────────────────────────────────── | |
| ax = axes_shape[0] | |
| y = stats.norm.pdf(x, 0, 1) | |
| ax.plot(x, y, color=C_BLUE, linewidth=2.5) | |
| ax.fill_between(x, y, alpha=ALPHA, color=C_BLUE) | |
| clean_ax(ax, "1.1 Bell Curve\n(Normal Distribution)") | |
| # ── 1.2 Uniform ───────────────────────────────────────────────────────────── | |
| ax = axes_shape[1] | |
| xu = np.linspace(-3, 3, 400) | |
| yu = stats.uniform.pdf(xu, -2.5, 5) | |
| ax.plot(xu, yu, color=C_GREEN, linewidth=2.5) | |
| ax.fill_between(xu, yu, alpha=ALPHA, color=C_GREEN) | |
| ax.set_ylim(bottom=0) | |
| clean_ax(ax, "1.2 Uniform\n(Flat / Even Distribution)") | |
| # ── 1.3 Skewness comparison ───────────────────────────────────────────────── | |
| ax = axes_shape[2] | |
| xr = np.linspace(0, 6, 400) | |
| # Right skew (positive) – lognormal | |
| yr = stats.lognorm.pdf(xr, s=0.8, scale=np.exp(0.9)) | |
| # Zero skew (symmetric) – normal centred at 3 | |
| yz = stats.norm.pdf(xr, 3, 0.85) | |
| # Left skew (negative) – reflected lognormal | |
| yl = stats.lognorm.pdf(6 - xr, s=0.8, scale=np.exp(0.9)) | |
| ax.plot(xr, yr, color=C_ORANGE, linewidth=2.2, label="Right skew (+)") | |
| ax.fill_between(xr, yr, alpha=ALPHA, color=C_ORANGE) | |
| ax.plot(xr, yz, color=C_BLUE, linewidth=2.2, label="Zero skew (symmetric)") | |
| ax.fill_between(xr, yz, alpha=ALPHA, color=C_BLUE) | |
| ax.plot(xr, yl, color=C_RED, linewidth=2.2, label="Left skew (−)") | |
| ax.fill_between(xr, yl, alpha=ALPHA, color=C_RED) | |
| ax.legend(fontsize=7.5, frameon=False, loc="upper right") | |
| clean_ax(ax, "1.3 Skewness\n(Right / Zero / Left)") | |
| # ── 1.4 (slot 4 kept for layout) – U-Shaped / Bimodal ────────────────────── | |
| ax = axes_shape[3] | |
| xb = np.linspace(-4, 4, 400) | |
| yb = stats.norm.pdf(xb, -2, 0.8) + stats.norm.pdf(xb, 2, 0.8) | |
| ax.plot(xb, yb, color=C_PURPLE, linewidth=2.5) | |
| ax.fill_between(xb, yb, alpha=ALPHA, color=C_PURPLE) | |
| clean_ax(ax, "1.5 U-Shaped\n(Bimodal Distribution)") | |
| # ── 1.5 Multimodal ────────────────────────────────────────────────────────── | |
| ax = axes_shape[4] | |
| xm = np.linspace(-5, 8, 600) | |
| ym = (stats.norm.pdf(xm, -3, 0.6) + | |
| 0.8 * stats.norm.pdf(xm, 0, 0.7) + | |
| 0.6 * stats.norm.pdf(xm, 3, 0.55) + | |
| 0.4 * stats.norm.pdf(xm, 6, 0.65)) | |
| ax.plot(xm, ym, color=C_TEAL, linewidth=2.5) | |
| ax.fill_between(xm, ym, alpha=ALPHA, color=C_TEAL) | |
| clean_ax(ax, "1.6 Multimodal\n(Multiple Peaks)") | |
| # ── unused 6th panel → off ─────────────────────────────────────────────────── | |
| axes_shape[5].axis("off") | |
| # ── Section B header + Central Tendency panel ──────────────────────────────── | |
| bottom = gridspec.GridSpecFromSubplotSpec( | |
| 2, 1, | |
| subplot_spec=outer[2], | |
| height_ratios=[0.18, 1], | |
| hspace=0.05 | |
| ) | |
| ax_head_B = fig.add_subplot(bottom[0]) | |
| ax_head_B.axis("off") | |
| ax_head_B.text(0.5, 0.5, "2 | Central Tendency", | |
| ha="center", va="center", | |
| fontsize=15, fontweight="bold", color="#2c2c2c", | |
| transform=ax_head_B.transAxes) | |
| ax_ct = fig.add_subplot(bottom[1]) | |
| # Right-skewed distribution (lognormal) for central tendency | |
| xct = np.linspace(0, 6, 600) | |
| yct = stats.lognorm.pdf(xct, s=0.7, scale=np.exp(0.7)) | |
| ax_ct.plot(xct, yct, color=C_BLUE, linewidth=2.5) | |
| ax_ct.fill_between(xct, yct, alpha=0.18, color=C_BLUE) | |
| # ── Calculate exact Mode, Median, Mean for this lognormal ────────────────── | |
| s_val, scale_val = 0.7, np.exp(0.7) | |
| mode_val = scale_val * np.exp(-s_val**2) # lognormal mode | |
| median_val = scale_val # lognormal median = scale | |
| mean_val = scale_val * np.exp(0.5 * s_val**2) # lognormal mean | |
| y_max = max(yct) | |
| # Vertical dashed lines | |
| def vline(ax_, xv, colour, label_): | |
| ax_.axvline(xv, color=colour, linewidth=2, linestyle="--") | |
| yval = stats.lognorm.pdf(xv, s=s_val, scale=scale_val) | |
| offset_y = y_max * 0.07 | |
| ax_.text(xv, yval + offset_y, label_, | |
| ha="center", va="bottom", | |
| fontsize=11, fontweight="bold", color=colour) | |
| vline(ax_ct, mode_val, C_GREEN, "Mode") | |
| vline(ax_ct, median_val, C_ORANGE, "Median") | |
| vline(ax_ct, mean_val, C_RED, "Mean") | |
| # Legend proxy lines | |
| from matplotlib.lines import Line2D | |
| legend_elements = [ | |
| Line2D([0], [0], color=C_GREEN, linewidth=2, linestyle="--", label=f"Mode ≈ {mode_val:.2f}"), | |
| Line2D([0], [0], color=C_ORANGE, linewidth=2, linestyle="--", label=f"Median ≈ {median_val:.2f}"), | |
| Line2D([0], [0], color=C_RED, linewidth=2, linestyle="--", label=f"Mean ≈ {mean_val:.2f}"), | |
| ] | |
| ax_ct.legend(handles=legend_elements, fontsize=10, frameon=False, | |
| loc="upper right", handlelength=2) | |
| for spine in ["top", "right", "left"]: | |
| ax_ct.spines[spine].set_visible(False) | |
| ax_ct.spines["bottom"].set_color("#AAAAAA") | |
| ax_ct.set_xticks([]) | |
| ax_ct.set_yticks([]) | |
| ax_ct.set_title("2.1 Mean, Median & Mode on a Right-Skewed Distribution", | |
| fontsize=12, fontweight="bold", pad=8) | |
| # Annotation arrow showing "tail pulls mean right" | |
| ax_ct.annotate( | |
| "Long right tail\npulls Mean higher", | |
| xy=(mean_val + 0.15, stats.lognorm.pdf(mean_val + 0.15, s=s_val, scale=scale_val)), | |
| xytext=(mean_val + 1.0, y_max * 0.45), | |
| fontsize=9, color="#555555", | |
| arrowprops=dict(arrowstyle="->", color="#888888", lw=1.2), | |
| ha="left" | |
| ) | |
| # # ── Save ───────────────────────────────────────────────────────────────────── | |
| # plt.savefig("distribution_diagrams.png", dpi=150, bbox_inches="tight", | |
| # facecolor="white") | |
| # plt.close() | |
| # print("Saved: distribution_diagrams.png") |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # # | |
| # Outliers & Anomaly Detection – Visualisations # | |
| # UTS 22804 Business Analytics Foundations – Week 9 # | |
| # Visualisation & Dashboarding # | |
| # # | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # | |
| # Figures (each rendered individually via .show()) | |
| # ───────────────────────────────────────────────── | |
| # | |
| # Section 1 – Time Series Anomalies | |
| # 1.1 Point Outliers | |
| # 1.2 Mean Shift (Structural Break) | |
| # 1.3 Temporary Mean Step | |
| # 1.4 Frequency Shift | |
| # 1.5 Variance / Noise Level Shift | |
| # 1.6 Missing Values Stored as Bad Dates | |
| # | |
| # Section 2 – Distribution Anomalies | |
| # 2.1 Sentinel Value: 999 ≈ NULL | |
| # 2.2 Sentinel Value: −1 ≈ NULL | |
| # | |
| # Section 3 – Dimensionality | |
| # 3.1 2D View – Outliers Hidden | |
| # 3.2 3D View – Outliers Revealed | |
| # | |
| # Section 4 – Other Anomaly Types | |
| # 4.1 Contextual Anomaly | |
| # 4.2 Bivariate Correlation Break | |
| # 4.3 Collective / Burst Anomaly | |
| # 4.4 Benford's Law Violation | |
| # 4.5 Anscombe III – One Outlier Distorts the Fit | |
| # 4.6 Anscombe IV – Leverage Point Creates an Illusion | |
| # | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # ── Install Dependencies (uncomment in Google Colab) ──────────────────────── | |
| # !pip install synthetic-data-generators plotly | |
| # ── Imports ───────────────────────────────────────────────────────────────── | |
| import numpy as np | |
| import pandas as pd | |
| import plotly.graph_objects as go | |
| from datetime import datetime | |
| from synthetic_data_generators.time_series import TimeSeriesGenerator | |
| # ── Settings ──────────────────────────────────────────────────────────────── | |
| SEED = 42 | |
| RNG = np.random.default_rng(SEED) | |
| TSG = TimeSeriesGenerator(seed=SEED) | |
| # ── Colour Palette ────────────────────────────────────────────────────────── | |
| C_BLUE = "#4C72B0" | |
| C_GREEN = "#55A868" | |
| C_ORANGE = "#DD8452" | |
| C_RED = "#C44E52" | |
| C_PURPLE = "#8172B2" | |
| C_TEAL = "#64B5CD" | |
| C_GREY = "#999999" | |
| # ── Common Layout Dict ───────────────────────────────────────────────────── | |
| LAYOUT = dict( | |
| template="simple_white", | |
| font=dict(family="Segoe UI, sans-serif", size=13), | |
| title_x=0.5, | |
| title_xanchor="center", | |
| paper_bgcolor="white", | |
| plot_bgcolor="white", | |
| margin=dict(t=100, b=55, l=60, r=40), | |
| width=1000, | |
| height=480, | |
| legend=dict( | |
| bgcolor="rgba(255,255,255,0.85)", | |
| bordercolor="rgba(0,0,0,0.1)", | |
| borderwidth=1, | |
| ), | |
| ) | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # Section 1 | Time Series Anomalies | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # ── 1.1 Point Outliers ───────────────────────────────────────────────────── | |
| # A regular seasonal pattern with a handful of injected anomalous spikes. | |
| # These represent sensor glitches, data-entry errors, or genuinely rare events. | |
| N1 = 730 | |
| outlier_idx_1 = [90, 260, 410, 605] | |
| df1 = TSG.create_time_series( | |
| start_date=datetime(2020, 1, 1), | |
| n_periods=N1, | |
| interpolation_nodes=[[0, 100], [N1, 100]], | |
| level_breaks=[], | |
| AR=[0.6], | |
| MA=[0], | |
| randomwalk_scale=1, | |
| noise_scale=4, | |
| season_conf={"style": "sin", "period_length": 30, "start_index": 0}, | |
| season_eff=0.3, | |
| manual_outliers=[[90, 280], [260, -60], [410, 300], [605, -80]], | |
| seed=SEED, | |
| ) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=df1["Date"], | |
| y=df1["Value"], | |
| mode="lines", | |
| name="Regular pattern", | |
| line=dict(color=C_TEAL, width=2), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=df1["Date"].iloc[outlier_idx_1], | |
| y=df1["Value"].iloc[outlier_idx_1], | |
| mode="markers", | |
| name="Point outlier", | |
| marker=dict(color=C_RED, size=11, symbol="x", line=dict(width=2, color=C_RED)), | |
| )) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "1.1 Point Outliers" | |
| "<br><sup>Regular seasonal pattern with occasional anomalous spikes</sup>" | |
| )), | |
| xaxis_title="Date", | |
| yaxis_title="Value", | |
| ) | |
| .show() | |
| ) | |
| # ── 1.2 Mean Shift (Structural Break) ───────────────────────────────────── | |
| # The underlying pattern continues unchanged, but the mean permanently jumps | |
| # to a new level. This can happen if a category is aggregated in a different | |
| # way, or some shock event changed the market conditions. | |
| N2 = 1096 | |
| bp2 = 500 | |
| df2 = TSG.create_time_series( | |
| start_date=datetime(2019, 1, 1), | |
| n_periods=N2, | |
| interpolation_nodes=[[0, 100], [N2, 100]], | |
| level_breaks=[[bp2, -40]], | |
| AR=[0.5], | |
| MA=[0], | |
| randomwalk_scale=0.5, | |
| noise_scale=3, | |
| season_conf={"style": "sin", "period_length": 30, "start_index": 0}, | |
| season_eff=0.15, | |
| manual_outliers=[], | |
| seed=SEED, | |
| ) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=df2["Date"], | |
| y=df2["Value"], | |
| mode="lines", | |
| name="Observed series", | |
| line=dict(color=C_TEAL, width=1.8), | |
| )) | |
| .add_shape(type="line", | |
| x0=df2["Date"].iloc[bp2], x1=df2["Date"].iloc[bp2], | |
| y0=0, y1=1, yref="paper", | |
| line=dict(dash="dash", color=C_RED, width=2), | |
| ) | |
| .add_annotation( | |
| x=df2["Date"].iloc[bp2], y=1, yref="paper", | |
| text="<b>Structural Break</b>", | |
| showarrow=False, xanchor="left", yanchor="top", | |
| font=dict(color=C_RED, size=11), | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "1.2 Mean Shift (Structural Break)" | |
| "<br><sup>A shock event causes the level to drop — same pattern continues from a new baseline</sup>" | |
| )), | |
| xaxis_title="Date", | |
| yaxis_title="Value", | |
| ) | |
| .show() | |
| ) | |
| # ── 1.3 Temporary Mean Step ─────────────────────────────────────────────── | |
| # The mean jumps up for a single cycle / window, then returns to its | |
| # previous level. Think of a one-off promotional campaign, a temporary | |
| # equipment fault, or a holiday-trading surge. | |
| N3 = 730 | |
| step_start, step_end = 320, 370 | |
| df3 = TSG.create_time_series( | |
| start_date=datetime(2020, 1, 1), | |
| n_periods=N3, | |
| interpolation_nodes=[[0, 100], [N3, 100]], | |
| level_breaks=[[step_start, 70], [step_end, -70]], | |
| AR=[0.5], | |
| MA=[0], | |
| randomwalk_scale=1, | |
| noise_scale=4, | |
| season_conf={"style": "sin", "period_length": 30, "start_index": 0}, | |
| season_eff=0.3, | |
| manual_outliers=[], | |
| seed=SEED, | |
| ) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=df3["Date"].iloc[:step_start], | |
| y=df3["Value"].iloc[:step_start], | |
| mode="lines", | |
| name="Normal", | |
| line=dict(color=C_TEAL, width=2), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=df3["Date"].iloc[step_start:step_end], | |
| y=df3["Value"].iloc[step_start:step_end], | |
| mode="lines", | |
| name="Anomalous period", | |
| line=dict(color=C_RED, width=2.5), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=df3["Date"].iloc[step_end:], | |
| y=df3["Value"].iloc[step_end:], | |
| mode="lines", | |
| name="Return to normal", | |
| line=dict(color=C_TEAL, width=2), | |
| showlegend=False, | |
| )) | |
| .add_shape(type="rect", | |
| x0=df3["Date"].iloc[step_start], x1=df3["Date"].iloc[step_end], | |
| y0=0, y1=1, yref="paper", | |
| fillcolor=C_RED, opacity=0.08, line_width=0, | |
| ) | |
| .add_annotation( | |
| x=df3["Date"].iloc[step_start], y=1, yref="paper", | |
| text="<b>Temporary Mean Step</b>", | |
| showarrow=False, xanchor="left", yanchor="top", | |
| font=dict(color=C_RED, size=11), | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "1.3 Temporary Mean Step" | |
| "<br><sup>Mean shifts up for one cycle, then drops back to normal</sup>" | |
| )), | |
| xaxis_title="Date", | |
| yaxis_title="Value", | |
| ) | |
| .show() | |
| ) | |
| # ── 1.4 Frequency Shift ────────────────────────────────────────────────── | |
| # The oscillation frequency changes after a structural break, while the mean | |
| # and amplitude stay roughly the same. Think of a manufacturing process | |
| # where the machine speed was reconfigured. | |
| N4 = 730 | |
| t4 = np.arange(N4) | |
| dates4 = pd.date_range("2020-01-01", periods=N4, freq="D") | |
| bp4 = 400 | |
| before4 = 100 + 15 * np.sin(2 * np.pi * t4[:bp4] / 60) + RNG.normal(0, 3, bp4) | |
| after4 = 100 + 15 * np.sin(2 * np.pi * t4[bp4:] / 15) + RNG.normal(0, 3, N4 - bp4) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=dates4, | |
| y=np.concatenate([before4, after4]), | |
| mode="lines", | |
| name="Observed series", | |
| line=dict(color=C_TEAL, width=1.8), | |
| )) | |
| .add_shape(type="line", | |
| x0=dates4[bp4], x1=dates4[bp4], | |
| y0=0, y1=1, yref="paper", | |
| line=dict(dash="dash", color=C_RED, width=2), | |
| ) | |
| .add_annotation( | |
| x=dates4[bp4], y=1, yref="paper", | |
| text="<b>Frequency Change</b>", | |
| showarrow=False, xanchor="left", yanchor="top", | |
| font=dict(color=C_RED, size=11), | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "1.4 Frequency Shift" | |
| "<br><sup>Same amplitude and mean, but oscillation frequency increases abruptly</sup>" | |
| )), | |
| xaxis_title="Date", | |
| yaxis_title="Value", | |
| ) | |
| .show() | |
| ) | |
| # ── 1.5 Variance / Noise Level Shift ───────────────────────────────────── | |
| # The trend continues but the noise fans out dramatically. We see such | |
| # patterns (or the reverse) in shipping rates when carriers started to apply | |
| # price-stabilising policies. | |
| N5 = 1096 | |
| bp5 = 500 | |
| t5 = np.arange(N5) | |
| dates5 = pd.date_range("2019-01-01", periods=N5, freq="D") | |
| trend5 = 70 + 0.12 * t5 + 0.00005 * t5**2 | |
| season5 = 12 * np.sin(2 * np.pi * t5 / 30) | |
| noise5 = np.concatenate([RNG.normal(0, 6, bp5), RNG.normal(0, 35, N5 - bp5)]) | |
| walk5 = np.cumsum(RNG.normal(0, 1.5, N5)) | |
| values5 = trend5 + season5 + noise5 + walk5 | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=dates5, | |
| y=values5, | |
| mode="lines", | |
| name="Observed series", | |
| line=dict(color=C_TEAL, width=1.8), | |
| )) | |
| .add_shape(type="line", | |
| x0=dates5[bp5], x1=dates5[bp5], | |
| y0=0, y1=1, yref="paper", | |
| line=dict(dash="dash", color=C_RED, width=2), | |
| ) | |
| .add_annotation( | |
| x=dates5[bp5], y=1, yref="paper", | |
| text="<b>Noise Level Change</b>", | |
| showarrow=False, xanchor="left", yanchor="top", | |
| font=dict(color=C_RED, size=11), | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "1.5 Variance / Noise Level Shift" | |
| "<br><sup>Same trend continues, but noise amplitude increases dramatically after the break</sup>" | |
| )), | |
| xaxis_title="Date", | |
| yaxis_title="Value", | |
| ) | |
| .show() | |
| ) | |
| # ── 1.6 Missing Values Stored as Bad Dates ──────────────────────────────── | |
| # Some legacy systems store NULL / missing timestamps as a default date | |
| # like '1 Jan 1990'. When you plot the raw data sorted by date, the | |
| # "missing" records cluster decades before the real data. | |
| N6 = 365 | |
| df6 = TSG.create_time_series( | |
| start_date=datetime(2020, 1, 1), | |
| n_periods=N6, | |
| interpolation_nodes=[[0, 100], [N6, 120]], | |
| level_breaks=[], | |
| AR=[0.5], | |
| MA=[0], | |
| randomwalk_scale=1, | |
| noise_scale=5, | |
| season_conf={"style": "sin", "period_length": 30, "start_index": 0}, | |
| season_eff=0.3, | |
| manual_outliers=[], | |
| seed=SEED, | |
| ) | |
| bad_idx_6 = [30, 62, 95, 160, 250, 290, 340] | |
| df6_plot = df6.copy().reset_index(drop=True) | |
| for i in bad_idx_6: | |
| df6_plot.at[i, "Date"] = datetime(1990, 1, 1) | |
| df6_plot = df6_plot.sort_values("Date").reset_index(drop=True) | |
| good_mask_6 = df6_plot["Date"] > datetime(2000, 1, 1) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=df6_plot.loc[good_mask_6, "Date"], | |
| y=df6_plot.loc[good_mask_6, "Value"], | |
| mode="lines+markers", | |
| name="Valid data", | |
| line=dict(color=C_TEAL, width=1.5), | |
| marker=dict(size=2), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=df6_plot.loc[~good_mask_6, "Date"], | |
| y=df6_plot.loc[~good_mask_6, "Value"], | |
| mode="markers", | |
| name="'Missing' → date = 1 Jan 1990", | |
| marker=dict(color=C_RED, size=11, symbol="x", line=dict(width=2, color=C_RED)), | |
| )) | |
| .add_annotation( | |
| x=datetime(1990, 1, 1), | |
| y=df6_plot.loc[~good_mask_6, "Value"].mean(), | |
| text="<b>Cluster at 1990-01-01</b><br>These are NULL values!", | |
| showarrow=True, arrowhead=2, arrowcolor=C_RED, | |
| font=dict(color=C_RED, size=11), | |
| ax=120, ay=-50, | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "1.6 Missing Values Stored as Bad Dates" | |
| "<br><sup>Some systems encode NULLs as a default date (e.g. 1 Jan 1990)</sup>" | |
| )), | |
| xaxis_title="Date", | |
| yaxis_title="Value", | |
| ) | |
| .show() | |
| ) | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # Section 2 | Distribution Anomalies | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # ── 2.1 Sentinel Value: 999 ≈ NULL ──────────────────────────────────────── | |
| # Many older databases and flat files store missing / empty values as a | |
| # large illogical number (999, 9999, -9999, etc.). | |
| real_21 = RNG.integers(1, 11, size=1000) | |
| null_21 = np.full(110, 999) | |
| all_21 = np.concatenate([real_21, null_21]) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Histogram( | |
| x=all_21, nbinsx=40, | |
| marker_color=C_BLUE, name="Values", | |
| )) | |
| .add_vline(x=999, line_dash="dash", line_color=C_RED, line_width=2) | |
| .add_annotation( | |
| x=999, y=110, | |
| text="<b>999 = NULL / Missing</b><br>110 records", | |
| showarrow=True, arrowhead=2, arrowcolor=C_RED, | |
| font=dict(color=C_RED, size=12), | |
| ax=-150, ay=-30, | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "2.1 Sentinel Value: 999 ≈ NULL" | |
| "<br><sup>Some databases store missing values as a large illogical number</sup>" | |
| )), | |
| xaxis_title="Value", | |
| yaxis_title="Count", | |
| bargap=0.05, | |
| showlegend=False, | |
| ) | |
| .show() | |
| ) | |
| # ── 2.2 Sentinel Value: −1 ≈ NULL ───────────────────────────────────────── | |
| # The sentinel is a negative number (−1) while the real data lives in a | |
| # completely different range (300–400). | |
| real_22 = RNG.uniform(300, 400, size=1200) | |
| null_22 = np.full(150, -1.0) | |
| all_22 = np.concatenate([real_22, null_22]) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Histogram( | |
| x=all_22, nbinsx=50, | |
| marker_color=C_GREEN, name="Values", | |
| )) | |
| .add_vline(x=-1, line_dash="dash", line_color=C_RED, line_width=2) | |
| .add_annotation( | |
| x=-1, y=150, | |
| text="<b>−1 = NULL / Missing</b><br>150 records", | |
| showarrow=True, arrowhead=2, arrowcolor=C_RED, | |
| font=dict(color=C_RED, size=12), | |
| ax=150, ay=-30, | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "2.2 Sentinel Value: −1 ≈ NULL" | |
| "<br><sup>Some systems store missing/empty values as −1 when normal range is 300–400</sup>" | |
| )), | |
| xaxis_title="Value", | |
| yaxis_title="Count", | |
| bargap=0.05, | |
| showlegend=False, | |
| ) | |
| .show() | |
| ) | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # Section 3 | Dimensionality – Outliers Hidden in Lower Dimensions | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # | |
| # Key lesson: some anomalies are invisible in a 2D projection but become | |
| # obvious when you visualise the data in 3D (or compute a multivariate | |
| # distance measure like Mahalanobis distance). | |
| n_norm_3d = 220 | |
| X1n = RNG.normal(0.0, 0.30, n_norm_3d) | |
| X2n = RNG.normal(0.0, 0.50, n_norm_3d) | |
| X3n = RNG.normal(0.5, 0.40, n_norm_3d) | |
| n_out_3d = 4 | |
| X1o = RNG.normal(0.3, 0.30, n_out_3d) | |
| X2o = RNG.normal(-0.2, 0.40, n_out_3d) | |
| X3o = RNG.uniform(-5.0, -3.0, n_out_3d) | |
| # ── 3.1 2D Scatter – Outliers Hidden ────────────────────────────────────── | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=X1n, y=X2n, | |
| mode="markers", | |
| name="Normal point", | |
| marker=dict(color=C_BLUE, size=7, opacity=0.65), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=X1o, y=X2o, | |
| mode="markers", | |
| name="Outlier (hidden!)", | |
| marker=dict(color=C_RED, size=11, symbol="diamond", opacity=0.9), | |
| )) | |
| .add_annotation( | |
| x=0.5, y=1.0, xref="paper", yref="paper", | |
| text="Can you spot the outliers? Neither can an algorithm looking at only 2 dimensions!", | |
| showarrow=False, | |
| font=dict(color=C_GREY, size=11), | |
| xanchor="center", | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "3.1 2D View (X₁ vs X₂) – Outliers are Invisible" | |
| "<br><sup>Looking at only two dimensions, the red points blend in perfectly</sup>" | |
| )), | |
| xaxis_title="X₁", | |
| yaxis_title="X₂", | |
| ) | |
| .show() | |
| ) | |
| # ── 3.2 3D Scatter – Outliers Revealed ──────────────────────────────────── | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter3d( | |
| x=X1n, y=X2n, z=X3n, | |
| mode="markers", | |
| name="Normal point", | |
| marker=dict(color=C_BLUE, size=3.5, opacity=0.65), | |
| )) | |
| .add_trace(go.Scatter3d( | |
| x=X1o, y=X2o, z=X3o, | |
| mode="markers", | |
| name="Outlier", | |
| marker=dict(color=C_RED, size=7, symbol="diamond", line=dict(width=1, color="black")), | |
| )) | |
| .update_layout( | |
| template="simple_white", | |
| font=dict(family="Segoe UI, sans-serif", size=13), | |
| title=dict( | |
| text=( | |
| "3.2 3D View (X₁, X₂, X₃) – Outliers Pop Out!" | |
| "<br><sup>Adding the third dimension reveals points far from the cluster</sup>" | |
| ), | |
| x=0.5, xanchor="center", | |
| ), | |
| paper_bgcolor="white", | |
| scene=dict( | |
| xaxis=dict(title="X₁", backgroundcolor="white", gridcolor="rgba(0,0,0,0.08)"), | |
| yaxis=dict(title="X₂", backgroundcolor="white", gridcolor="rgba(0,0,0,0.08)"), | |
| zaxis=dict(title="X₃", backgroundcolor="white", gridcolor="rgba(0,0,0,0.08)"), | |
| camera=dict(eye=dict(x=1.6, y=1.5, z=0.85)), | |
| ), | |
| legend=dict( | |
| bgcolor="rgba(255,255,255,0.85)", | |
| bordercolor="rgba(0,0,0,0.1)", | |
| borderwidth=1, | |
| ), | |
| width=1000, | |
| height=620, | |
| margin=dict(l=30, r=30, t=90, b=30), | |
| ) | |
| .show() | |
| ) | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # Section 4 | Other Anomaly Types | |
| # ═══════════════════════════════════════════════════════════════════════════════ | |
| # ── 4.1 Contextual Anomaly ──────────────────────────────────────────────── | |
| # A value that is perfectly normal in one context (e.g. $56 revenue at noon) | |
| # but anomalous in another (e.g. $56 at 3am when revenue should be ~$22). | |
| # Key lesson: "outlier" depends on context, not just magnitude. | |
| dates_ctx = pd.date_range("2024-01-01", periods=14 * 24, freq="h") | |
| hour_ctx = dates_ctx.hour.values | |
| daily_cycle = 40 + 18 * np.sin(2 * np.pi * hour_ctx / 24 - np.pi / 2) | |
| values_ctx = daily_cycle + RNG.normal(0, 2.5, len(dates_ctx)) | |
| anomaly_idx_ctx = 6 * 24 + 3 # 3am on day 7 | |
| values_ctx[anomaly_idx_ctx] = 56 | |
| df_ctx = pd.DataFrame({"date": dates_ctx, "value": values_ctx}) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=df_ctx["date"], y=df_ctx["value"], | |
| mode="lines", | |
| name="Regular daily pattern", | |
| line=dict(color=C_TEAL, width=2), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=[df_ctx.loc[anomaly_idx_ctx, "date"]], | |
| y=[df_ctx.loc[anomaly_idx_ctx, "value"]], | |
| mode="markers", | |
| name="Contextual anomaly", | |
| marker=dict(color=C_RED, size=12, symbol="diamond"), | |
| )) | |
| .add_annotation( | |
| x=df_ctx.loc[anomaly_idx_ctx, "date"], | |
| y=df_ctx.loc[anomaly_idx_ctx, "value"], | |
| text="<b>$56 at 3am</b><br>Looks ordinary overall,<br>but very odd for this time of day", | |
| showarrow=True, arrowhead=2, arrowcolor=C_RED, | |
| ax=100, ay=-60, | |
| bgcolor="rgba(255,255,255,0.88)", | |
| bordercolor=C_RED, borderwidth=1, | |
| font=dict(size=11), | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "4.1 Contextual Anomaly" | |
| "<br><sup>$56 is normal at midday — but suspicious at 3am when revenue should be ~$22</sup>" | |
| )), | |
| xaxis_title="Date / Time", | |
| yaxis_title="Revenue ($)", | |
| ) | |
| .show() | |
| ) | |
| # ── 4.2 Bivariate Correlation Break ────────────────────────────────────── | |
| # Points that are individually within range on both axes, but fall off the | |
| # expected relationship between the two variables. | |
| rng_bv = np.random.default_rng(42) | |
| x_bv = rng_bv.uniform(0, 100, 220) | |
| y_bv = 2.2 * x_bv + 20 + rng_bv.normal(0, 10, len(x_bv)) | |
| x_bv_out = np.array([25, 40, 72, 88]) | |
| y_bv_out = np.array([145, 45, 115, 68]) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=x_bv, y=y_bv, | |
| mode="markers", | |
| name="Expected relationship", | |
| marker=dict(color=C_BLUE, size=7, opacity=0.7), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=x_bv_out, y=y_bv_out, | |
| mode="markers", | |
| name="Relationship outlier", | |
| marker=dict(color=C_RED, size=11, symbol="diamond", opacity=0.9), | |
| )) | |
| .add_annotation( | |
| x=0.98, y=0.05, xref="paper", yref="paper", | |
| text="Each red point is within normal<br>range on both axes individually,<br>but breaks the relationship", | |
| showarrow=False, | |
| bgcolor="rgba(255,255,255,0.88)", | |
| bordercolor=C_RED, borderwidth=1, | |
| font=dict(color=C_RED, size=11), | |
| xanchor="right", | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "4.2 Bivariate Correlation Break" | |
| "<br><sup>Points that break the otherwise strong linear relationship</sup>" | |
| )), | |
| xaxis_title="Input Variable", | |
| yaxis_title="Output Variable", | |
| ) | |
| .show() | |
| ) | |
| # ── 4.3 Collective / Burst Anomaly ─────────────────────────────────────── | |
| # A short run of consecutive points that are individually not extreme, but | |
| # together form an unusual pattern — e.g. a run of higher-than-expected | |
| # values, or a flat-line in an otherwise fluctuating signal. | |
| N_burst = 500 | |
| t_burst = np.arange(N_burst) | |
| dates_burst = pd.date_range("2023-01-01", periods=N_burst, freq="D") | |
| base_burst = 80 + 8 * np.sin(2 * np.pi * t_burst / 24) + RNG.normal(0, 1.8, N_burst) | |
| burst_start, burst_end = 260, 290 | |
| values_burst = base_burst.copy() | |
| values_burst[burst_start:burst_end] += np.linspace(14, 26, burst_end - burst_start) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=dates_burst, y=values_burst, | |
| mode="lines", | |
| name="Observed series", | |
| line=dict(color=C_TEAL, width=2), | |
| )) | |
| .add_shape(type="rect", | |
| x0=dates_burst[burst_start], x1=dates_burst[burst_end], | |
| y0=0, y1=1, yref="paper", | |
| fillcolor=C_RED, opacity=0.12, line_width=0, | |
| ) | |
| .add_annotation( | |
| x=dates_burst[burst_start], y=1, yref="paper", | |
| text="<b>Anomalous Burst</b>", | |
| showarrow=False, xanchor="left", yanchor="top", | |
| font=dict(color=C_RED, size=11), | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "4.3 Collective / Burst Anomaly" | |
| "<br><sup>Each point is individually plausible, but the consecutive run is collectively unusual</sup>" | |
| )), | |
| xaxis_title="Date", | |
| yaxis_title="Value", | |
| ) | |
| .show() | |
| ) | |
| # ── 4.4 Benford's Law Violation ────────────────────────────────────────── | |
| # In naturally-occurring numeric data, the leading digit follows a | |
| # logarithmic distribution (Benford's Law). Fabricated or manipulated data | |
| # often violates this — a classic forensic-accounting red flag. | |
| digits = np.arange(1, 10) | |
| benford_pct = np.log10(1 + 1 / digits) * 100 | |
| # Natural data: log-uniform produces Benford-compliant leading digits | |
| natural_data = 10 ** RNG.uniform(1, 6, size=10000) | |
| leading_natural = np.array([int(str(abs(v))[0]) for v in natural_data]) | |
| natural_pct = np.array([(leading_natural == d).sum() for d in digits]) | |
| natural_pct = natural_pct / natural_pct.sum() * 100 | |
| # Fabricated data: people tend to pick leading digits too uniformly | |
| fabricated_weights = np.array([0.14, 0.06, 0.08, 0.10, 0.16, 0.14, 0.15, 0.09, 0.08]) | |
| fabricated_pct = fabricated_weights / fabricated_weights.sum() * 100 | |
| ( | |
| go.Figure() | |
| .add_trace(go.Bar( | |
| x=digits, y=benford_pct, | |
| name="Benford's Law (expected)", | |
| marker_color=C_BLUE, opacity=0.85, | |
| )) | |
| .add_trace(go.Bar( | |
| x=digits, y=natural_pct, | |
| name="Natural data (observed)", | |
| marker_color=C_GREEN, opacity=0.75, | |
| )) | |
| .add_trace(go.Bar( | |
| x=digits, y=fabricated_pct, | |
| name="Fabricated data (suspicious!)", | |
| marker_color=C_RED, opacity=0.75, | |
| )) | |
| .add_annotation( | |
| x=5, y=max(fabricated_pct) + 2, | |
| text="<b>Fabricated data deviates</b><br>from the expected distribution", | |
| showarrow=True, arrowhead=2, arrowcolor=C_RED, | |
| ax=0, ay=-45, | |
| font=dict(color=C_RED, size=11), | |
| bgcolor="rgba(255,255,255,0.88)", | |
| bordercolor=C_RED, borderwidth=1, | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| barmode="group", | |
| title=dict(text=( | |
| "4.4 Benford's Law Violation" | |
| "<br><sup>Natural data follows a logarithmic leading-digit distribution — fabricated data doesn't</sup>" | |
| )), | |
| xaxis_title="Leading Digit", | |
| yaxis_title="Frequency (%)", | |
| ) | |
| .update_xaxes(dtick=1) | |
| .show() | |
| ) | |
| # ── 4.5 Anscombe III – One Outlier Distorts the Fit ─────────────────────── | |
| # Anscombe's Quartet dataset III: a near-perfect linear relationship except | |
| # for a single high-leverage outlier that pulls the regression line away | |
| # from the true pattern. The summary statistics (mean, variance, correlation, | |
| # regression slope) are identical across all four Anscombe datasets — yet the | |
| # data tell very different stories. | |
| x_a3 = np.array([10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5], dtype=float) | |
| y_a3 = np.array([7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]) | |
| # OLS fit (all points) | |
| slope_a3, intercept_a3 = np.polyfit(x_a3, y_a3, 1) | |
| fit_x_a3 = np.array([x_a3.min() - 0.5, x_a3.max() + 0.5]) | |
| fit_y_a3 = slope_a3 * fit_x_a3 + intercept_a3 | |
| outlier_mask_a3 = (x_a3 == 13) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=x_a3[~outlier_mask_a3], y=y_a3[~outlier_mask_a3], | |
| mode="markers", | |
| name="Normal points", | |
| marker=dict(color=C_GREEN, size=9, opacity=0.85), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=x_a3[outlier_mask_a3], y=y_a3[outlier_mask_a3], | |
| mode="markers", | |
| name="Outlier", | |
| marker=dict(color=C_RED, size=12, symbol="diamond"), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=fit_x_a3, y=fit_y_a3, | |
| mode="lines", | |
| name=f"OLS fit (slope = {slope_a3:.2f})", | |
| line=dict(color=C_GREEN, width=2), | |
| )) | |
| .add_annotation( | |
| x=13, y=12.74, | |
| text="<b>Single outlier</b><br>pulls the regression line<br>away from the true pattern", | |
| showarrow=True, arrowhead=2, arrowcolor=C_RED, | |
| ax=-120, ay=-40, | |
| bgcolor="rgba(255,255,255,0.88)", | |
| bordercolor=C_RED, borderwidth=1, | |
| font=dict(color=C_RED, size=11), | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "4.5 Anscombe III – One Outlier Distorts the Fit" | |
| "<br><sup>A single anomalous point pulls the regression line away from the otherwise tight pattern</sup>" | |
| )), | |
| xaxis_title="x", | |
| yaxis_title="y", | |
| ) | |
| .show() | |
| ) | |
| # ── 4.6 Anscombe IV – Leverage Point Creates an Illusion ───────────────── | |
| # Anscombe's Quartet dataset IV: all x-values are identical (x=0) except one | |
| # point at x=1. That single leverage point single-handedly manufactures a | |
| # linear slope — without it, there is no relationship at all. | |
| x_a4 = np.array([0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], dtype=float) | |
| y_a4 = np.array([6.58, 5.76, 7.71, 8.84, 8.47, 7.04, 5.25, 12.50, 5.56, 7.91, 6.89]) | |
| slope_a4, intercept_a4 = np.polyfit(x_a4, y_a4, 1) | |
| fit_x_a4 = np.array([x_a4.min() - 0.5, x_a4.max() + 0.5]) | |
| fit_y_a4 = slope_a4 * fit_x_a4 + intercept_a4 | |
| lever_mask_a4 = (x_a4 == 1) | |
| ( | |
| go.Figure() | |
| .add_trace(go.Scatter( | |
| x=x_a4[~lever_mask_a4], y=y_a4[~lever_mask_a4], | |
| mode="markers", | |
| name="Clustered points (x = 0)", | |
| marker=dict(color=C_ORANGE, size=9, opacity=0.85), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=x_a4[lever_mask_a4], y=y_a4[lever_mask_a4], | |
| mode="markers", | |
| name="Leverage point", | |
| marker=dict(color=C_RED, size=12, symbol="diamond"), | |
| )) | |
| .add_trace(go.Scatter( | |
| x=fit_x_a4, y=fit_y_a4, | |
| mode="lines", | |
| name=f"OLS fit (slope = {slope_a4:.2f})", | |
| line=dict(color=C_RED, width=2, dash="dash"), | |
| )) | |
| .add_annotation( | |
| x=1, y=12.50, | |
| text="<b>Single leverage point</b><br>manufactures a slope<br>that doesn't really exist", | |
| showarrow=True, arrowhead=2, arrowcolor=C_RED, | |
| ax=-140, ay=-30, | |
| bgcolor="rgba(255,255,255,0.88)", | |
| bordercolor=C_RED, borderwidth=1, | |
| font=dict(color=C_RED, size=11), | |
| ) | |
| .update_layout( | |
| **LAYOUT, | |
| title=dict(text=( | |
| "4.6 Anscombe IV – Leverage Point Creates an Illusion" | |
| "<br><sup>One extreme x-value single-handedly manufactures a linear relationship</sup>" | |
| )), | |
| xaxis_title="x", | |
| yaxis_title="y", | |
| ) | |
| .show() | |
| ) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment