Skip to content

Instantly share code, notes, and snippets.

@nmicic

nmicic/microgpt.c

Forked from karpathy/microgpt.py
Last active March 23, 2026 16:43
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save nmicic/35316463f3c5e8e9fe8eb599b3842b58 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save nmicic/35316463f3c5e8e9fe8eb599b3842b58 to your computer and use it in GitHub Desktop.
Download ZIP
microgpt.c — sub-20ms GPT training and inference in pure C
Raw
microgpt.c
/* Copyright (c) 2026 Nenad Mićić <nenad@micic.be> https://be.linkedin.com/in/nenadmicic
* SPDX-License-Identifier: Apache-2.0
*
* microgpt.c — Minimal GPT training and inference in 655 lines of pure C.
* Hand-written forward and backward pass, two-phase gradient accumulation.
* Trains a 1-layer, 4-head, 4192-parameter character-level transformer.
*
* Inspired by Andrej Karpathy's microgpt.py (200 lines of Python):
* https://gist.github.com/karpathy/8627fe009c40f57531cb18360106ce95
*
* Single-file, no dependencies beyond libc + libm.
* Compile: cc -O2 -Wall -Wextra -ffp-contract=off -o microgpt microgpt.c -lm
*
* Dataset: 32K names from karpathy/makemore (lowercase a-z only):
* https://raw.githubusercontent.com/karpathy/makemore/master/names.txt
* Save as names.txt in the same directory.
*
* Performance: ~19 ms on a MacBook Air M3 for 1000 training steps + 20 samples,
* compiled with cc -O2 -Wall -Wextra -ffp-contract=off. * Bit-exact output match with the Python reference.
*
* For datasets with characters beyond a-z, the tokenizer needs to be
* made dynamic (build vocab from sorted unique chars at runtime).
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdint.h>
/* ── Model constants ─────────────────────────────────────────────── */
#define N_EMBD 16
#define N_HEAD 4
#define HEAD_DIM (N_EMBD / N_HEAD)
#define BLOCK_SIZE 16
#define N_LAYER 1
#define MLP_DIM (4 * N_EMBD)
#define VOCAB_SIZE 27
#define NUM_PARAMS 4192
#define MAX_DOCS 33000
#define MAX_NAME 32
/* ── MT19937 RNG ─────────────────────────────────────────────────── */
#define MT_N 624
#define MT_M 397
#define MT_MATRIX_A 0x9908b0dfU
#define MT_UPPER 0x80000000U
#define MT_LOWER 0x7fffffffU
static uint32_t mt[MT_N];
static int mt_idx = MT_N + 1;
static void mt_init_genrand(uint32_t s) {
mt[0] = s;
for (int i = 1; i < MT_N; i++)
mt[i] = 1812433253U * (mt[i-1] ^ (mt[i-1] >> 30)) + (uint32_t)i;
mt_idx = MT_N;
}
static void mt_init_by_array(const uint32_t *key, int keylen) {
mt_init_genrand(19650218U);
int i = 1, j = 0;
int k = MT_N > keylen ? MT_N : keylen;
for (; k; k--) {
mt[i] = (mt[i] ^ ((mt[i-1] ^ (mt[i-1] >> 30)) * 1664525U))
+ key[j] + (uint32_t)j;
i++; j++;
if (i >= MT_N) { mt[0] = mt[MT_N-1]; i = 1; }
if (j >= keylen) j = 0;
}
for (k = MT_N - 1; k; k--) {
mt[i] = (mt[i] ^ ((mt[i-1] ^ (mt[i-1] >> 30)) * 1566083941U))
- (uint32_t)i;
i++;
if (i >= MT_N) { mt[0] = mt[MT_N-1]; i = 1; }
}
mt[0] = 0x80000000U;
mt_idx = MT_N;
}
static uint32_t mt_genrand(void) {
static const uint32_t mag01[2] = {0U, MT_MATRIX_A};
if (mt_idx >= MT_N) {
int kk;
for (kk = 0; kk < MT_N - MT_M; kk++) {
uint32_t y = (mt[kk] & MT_UPPER) | (mt[kk+1] & MT_LOWER);
mt[kk] = mt[kk + MT_M] ^ (y >> 1) ^ mag01[y & 1U];
}
for (; kk < MT_N - 1; kk++) {
uint32_t y = (mt[kk] & MT_UPPER) | (mt[kk+1] & MT_LOWER);
mt[kk] = mt[kk + (MT_M - MT_N)] ^ (y >> 1) ^ mag01[y & 1U];
}
{
uint32_t y = (mt[MT_N-1] & MT_UPPER) | (mt[0] & MT_LOWER);
mt[MT_N-1] = mt[MT_M-1] ^ (y >> 1) ^ mag01[y & 1U];
}
mt_idx = 0;
}
uint32_t y = mt[mt_idx++];
y ^= (y >> 11);
y ^= (y << 7) & 0x9d2c5680U;
y ^= (y << 15) & 0xefc60000U;
y ^= (y >> 18);
return y;
}
static double rng_double(void) {
uint32_t a = mt_genrand() >> 5;
uint32_t b = mt_genrand() >> 6;
return (a * 67108864.0 + b) * (1.0 / 9007199254740992.0);
}
static double gauss_next_val;
static int gauss_has_next = 0;
static double rng_gauss(double mu, double sigma) {
double z;
if (gauss_has_next) {
z = gauss_next_val;
gauss_has_next = 0;
} else {
double x2pi = rng_double() * (2.0 * M_PI);
double g2rad = sqrt(-2.0 * log(1.0 - rng_double()));
z = cos(x2pi) * g2rad;
gauss_next_val = sin(x2pi) * g2rad;
gauss_has_next = 1;
}
return mu + z * sigma;
}
static uint32_t rng_getrandbits(int k) {
return mt_genrand() >> (32 - k);
}
static int bit_length(uint32_t n) {
int b = 0;
while (n) { b++; n >>= 1; }
return b;
}
static int rng_randbelow(int n) {
int k = bit_length((uint32_t)n);
int r = (int)rng_getrandbits(k);
while (r >= n) r = (int)rng_getrandbits(k);
return r;
}
static void rng_shuffle(int *arr, int len) {
for (int i = len - 1; i >= 1; i--) {
int j = rng_randbelow(i + 1);
int tmp = arr[i]; arr[i] = arr[j]; arr[j] = tmp;
}
}
static int rng_weighted_choice(const double *weights, int n) {
double cum[VOCAB_SIZE];
cum[0] = weights[0];
for (int i = 1; i < n; i++) cum[i] = cum[i-1] + weights[i];
double total = cum[n-1];
double target = rng_double() * total;
int lo = 0, hi = n - 1;
while (lo < hi) {
int mid = (lo + hi) / 2;
if (target < cum[mid]) hi = mid;
else lo = mid + 1;
}
return lo;
}
/* ── Data ────────────────────────────────────────────────────────── */
static char docs[MAX_DOCS][MAX_NAME];
static int doc_idx[MAX_DOCS];
static int num_docs;
static void load_data(void) {
FILE *f = fopen("names.txt", "r");
if (!f) { fprintf(stderr, "Cannot open names.txt\n"); exit(1); }
char buf[256];
num_docs = 0;
while (fgets(buf, sizeof(buf), f)) {
int len = (int)strlen(buf);
while (len > 0 && (buf[len-1] == '\n' || buf[len-1] == '\r' || buf[len-1] == ' '))
buf[--len] = '\0';
if (len == 0) continue;
if (num_docs >= MAX_DOCS) { fprintf(stderr, "Too many docs\n"); exit(1); }
if (len >= MAX_NAME) len = MAX_NAME - 1;
memcpy(docs[num_docs], buf, len);
docs[num_docs][len] = '\0';
doc_idx[num_docs] = num_docs;
num_docs++;
}
fclose(f);
}
/* ── Model parameters ────────────────────────────────────────────── */
/*
* Layout (row-major):
* wte: VOCAB_SIZE x N_EMBD = 27*16 = 432
* wpe: BLOCK_SIZE x N_EMBD = 16*16 = 256
* lm_head: VOCAB_SIZE x N_EMBD = 27*16 = 432
* wq: N_EMBD x N_EMBD = 16*16 = 256
* wk: N_EMBD x N_EMBD = 16*16 = 256
* wv: N_EMBD x N_EMBD = 16*16 = 256
* wo: N_EMBD x N_EMBD = 16*16 = 256
* fc1: MLP_DIM x N_EMBD = 64*16 = 1024
* fc2: N_EMBD x MLP_DIM = 16*64 = 1024
* Total: 4192
*/
static double params[NUM_PARAMS];
static double grads[NUM_PARAMS];
static double adam_m[NUM_PARAMS];
static double adam_v[NUM_PARAMS];
#define OFF_WTE 0
#define OFF_WPE (OFF_WTE + VOCAB_SIZE * N_EMBD)
#define OFF_LMHEAD (OFF_WPE + BLOCK_SIZE * N_EMBD)
#define OFF_WQ (OFF_LMHEAD + VOCAB_SIZE * N_EMBD)
#define OFF_WK (OFF_WQ + N_EMBD * N_EMBD)
#define OFF_WV (OFF_WK + N_EMBD * N_EMBD)
#define OFF_WO (OFF_WV + N_EMBD * N_EMBD)
#define OFF_FC1 (OFF_WO + N_EMBD * N_EMBD)
#define OFF_FC2 (OFF_FC1 + MLP_DIM * N_EMBD)
#define W(off, r, c, cols) params[(off) + (r) * (cols) + (c)]
#define G(off, r, c, cols) grads[(off) + (r) * (cols) + (c)]
/* ── Forward-pass intermediates ──────────────────────────────────── */
static double fwd_emb [BLOCK_SIZE][N_EMBD];
static double fwd_x0 [BLOCK_SIZE][N_EMBD];
static double fwd_x0_scale[BLOCK_SIZE];
static double fwd_xr1 [BLOCK_SIZE][N_EMBD];
static double fwd_xn1 [BLOCK_SIZE][N_EMBD];
static double fwd_xn1_sc [BLOCK_SIZE];
static double fwd_q [BLOCK_SIZE][N_EMBD];
static double fwd_k [BLOCK_SIZE][N_EMBD];
static double fwd_v [BLOCK_SIZE][N_EMBD];
static double fwd_attn_w [BLOCK_SIZE][N_HEAD][BLOCK_SIZE];
static double fwd_xa [BLOCK_SIZE][N_EMBD];
static double fwd_x2 [BLOCK_SIZE][N_EMBD];
static double fwd_xn2 [BLOCK_SIZE][N_EMBD];
static double fwd_xn2_sc [BLOCK_SIZE];
static double fwd_fc1pre [BLOCK_SIZE][MLP_DIM];
static double fwd_fc1post[BLOCK_SIZE][MLP_DIM];
static double fwd_x3 [BLOCK_SIZE][N_EMBD];
static double fwd_logits [BLOCK_SIZE][VOCAB_SIZE];
static double fwd_probs [BLOCK_SIZE][VOCAB_SIZE];
/* ── Forward helpers ─────────────────────────────────────────────── */
static double do_rmsnorm(const double *x, double *y, int D) {
double ms = 0;
for (int i = 0; i < D; i++) ms += x[i] * x[i];
ms /= D;
double scale = 1.0 / sqrt(ms + 1e-5);
for (int i = 0; i < D; i++) y[i] = x[i] * scale;
return scale;
}
static void do_linear(const double *x, int off, int nout, int nin, double *y) {
for (int r = 0; r < nout; r++) {
double s = 0;
for (int c = 0; c < nin; c++)
s += params[off + r * nin + c] * x[c];
y[r] = s;
}
}
static void do_softmax(const double *in, double *out, int n) {
double mx = in[0];
for (int i = 1; i < n; i++) if (in[i] > mx) mx = in[i];
double total = 0;
for (int i = 0; i < n; i++) { out[i] = exp(in[i] - mx); total += out[i]; }
for (int i = 0; i < n; i++) out[i] /= total;
}
/* ── Training forward pass ───────────────────────────────────────── */
static double train_forward(const int *tokens, int n) {
for (int t = 0; t < n; t++) {
int tok = tokens[t];
int pos = t;
for (int i = 0; i < N_EMBD; i++)
fwd_emb[t][i] = W(OFF_WTE, tok, i, N_EMBD) + W(OFF_WPE, pos, i, N_EMBD);
fwd_x0_scale[t] = do_rmsnorm(fwd_emb[t], fwd_x0[t], N_EMBD);
memcpy(fwd_xr1[t], fwd_x0[t], N_EMBD * sizeof(double));
fwd_xn1_sc[t] = do_rmsnorm(fwd_x0[t], fwd_xn1[t], N_EMBD);
do_linear(fwd_xn1[t], OFF_WQ, N_EMBD, N_EMBD, fwd_q[t]);
do_linear(fwd_xn1[t], OFF_WK, N_EMBD, N_EMBD, fwd_k[t]);
do_linear(fwd_xn1[t], OFF_WV, N_EMBD, N_EMBD, fwd_v[t]);
double inv_sqrt_hd = 1.0 / sqrt((double)HEAD_DIM);
for (int h = 0; h < N_HEAD; h++) {
int hs = h * HEAD_DIM;
double attn_logits[BLOCK_SIZE];
for (int s = 0; s <= t; s++) {
double dot = 0;
for (int j = 0; j < HEAD_DIM; j++)
dot += fwd_q[t][hs+j] * fwd_k[s][hs+j];
attn_logits[s] = dot * inv_sqrt_hd;
}
double mx = attn_logits[0];
for (int s = 1; s <= t; s++) if (attn_logits[s] > mx) mx = attn_logits[s];
double total = 0;
for (int s = 0; s <= t; s++) {
fwd_attn_w[t][h][s] = exp(attn_logits[s] - mx);
total += fwd_attn_w[t][h][s];
}
for (int s = 0; s <= t; s++) fwd_attn_w[t][h][s] /= total;
for (int j = 0; j < HEAD_DIM; j++) {
double acc = 0;
for (int s = 0; s <= t; s++)
acc += fwd_attn_w[t][h][s] * fwd_v[s][hs+j];
fwd_xa[t][hs+j] = acc;
}
}
double wo_out[N_EMBD];
do_linear(fwd_xa[t], OFF_WO, N_EMBD, N_EMBD, wo_out);
for (int i = 0; i < N_EMBD; i++)
fwd_x2[t][i] = wo_out[i] + fwd_xr1[t][i];
fwd_xn2_sc[t] = do_rmsnorm(fwd_x2[t], fwd_xn2[t], N_EMBD);
do_linear(fwd_xn2[t], OFF_FC1, MLP_DIM, N_EMBD, fwd_fc1pre[t]);
for (int i = 0; i < MLP_DIM; i++)
fwd_fc1post[t][i] = fwd_fc1pre[t][i] > 0 ? fwd_fc1pre[t][i] : 0;
double fc2_out[N_EMBD];
do_linear(fwd_fc1post[t], OFF_FC2, N_EMBD, MLP_DIM, fc2_out);
for (int i = 0; i < N_EMBD; i++)
fwd_x3[t][i] = fc2_out[i] + fwd_x2[t][i];
do_linear(fwd_x3[t], OFF_LMHEAD, VOCAB_SIZE, N_EMBD, fwd_logits[t]);
do_softmax(fwd_logits[t], fwd_probs[t], VOCAB_SIZE);
}
double loss = 0;
for (int t = 0; t < n; t++) {
int target = tokens[t + 1];
loss += -log(fwd_probs[t][target]);
}
return loss / n;
}
/* ── Backward pass (two-phase) ───────────────────────────────────── */
static double bwd_dk[BLOCK_SIZE][N_EMBD];
static double bwd_dv[BLOCK_SIZE][N_EMBD];
static double bwd_d_xr1[BLOCK_SIZE][N_EMBD];
static void train_backward(const int *tokens, int n) {
memset(grads, 0, sizeof(grads));
memset(bwd_dk, 0, sizeof(bwd_dk));
memset(bwd_dv, 0, sizeof(bwd_dv));
double d_q[BLOCK_SIZE][N_EMBD];
/* Phase 1 (reverse): loss → lm_head → MLP → attention */
for (int t = n - 1; t >= 0; t--) {
int target = tokens[t + 1];
double d_logits[VOCAB_SIZE];
for (int i = 0; i < VOCAB_SIZE; i++)
d_logits[i] = (fwd_probs[t][i] - (i == target ? 1.0 : 0.0)) / n;
double d_x3[N_EMBD];
memset(d_x3, 0, sizeof(d_x3));
for (int r = 0; r < VOCAB_SIZE; r++) {
for (int c = 0; c < N_EMBD; c++) {
G(OFF_LMHEAD, r, c, N_EMBD) += d_logits[r] * fwd_x3[t][c];
d_x3[c] += d_logits[r] * params[OFF_LMHEAD + r * N_EMBD + c];
}
}
double d_x2[N_EMBD];
memcpy(d_x2, d_x3, sizeof(d_x2));
double d_fc1post[MLP_DIM];
memset(d_fc1post, 0, sizeof(d_fc1post));
for (int r = 0; r < N_EMBD; r++) {
for (int c = 0; c < MLP_DIM; c++) {
G(OFF_FC2, r, c, MLP_DIM) += d_x3[r] * fwd_fc1post[t][c];
d_fc1post[c] += d_x3[r] * params[OFF_FC2 + r * MLP_DIM + c];
}
}
double d_fc1pre[MLP_DIM];
for (int i = 0; i < MLP_DIM; i++)
d_fc1pre[i] = fwd_fc1pre[t][i] > 0 ? d_fc1post[i] : 0;
double d_xn2[N_EMBD];
memset(d_xn2, 0, sizeof(d_xn2));
for (int r = 0; r < MLP_DIM; r++) {
for (int c = 0; c < N_EMBD; c++) {
G(OFF_FC1, r, c, N_EMBD) += d_fc1pre[r] * fwd_xn2[t][c];
d_xn2[c] += d_fc1pre[r] * params[OFF_FC1 + r * N_EMBD + c];
}
}
{
double sc = fwd_xn2_sc[t];
double dot = 0;
for (int i = 0; i < N_EMBD; i++) dot += d_xn2[i] * fwd_x2[t][i];
double sc3 = sc * sc * sc;
double coeff = sc3 * dot / N_EMBD;
for (int i = 0; i < N_EMBD; i++)
d_x2[i] += sc * d_xn2[i] - fwd_x2[t][i] * coeff;
}
memcpy(bwd_d_xr1[t], d_x2, N_EMBD * sizeof(double));
double d_xa[N_EMBD];
memset(d_xa, 0, sizeof(d_xa));
for (int r = 0; r < N_EMBD; r++) {
for (int c = 0; c < N_EMBD; c++) {
G(OFF_WO, r, c, N_EMBD) += d_x2[r] * fwd_xa[t][c];
d_xa[c] += d_x2[r] * params[OFF_WO + r * N_EMBD + c];
}
}
double inv_sqrt_hd = 1.0 / sqrt((double)HEAD_DIM);
memset(d_q[t], 0, sizeof(d_q[t]));
for (int h = 0; h < N_HEAD; h++) {
int hs = h * HEAD_DIM;
double d_attn_w[BLOCK_SIZE];
memset(d_attn_w, 0, sizeof(d_attn_w));
for (int s = 0; s <= t; s++) {
for (int j = 0; j < HEAD_DIM; j++) {
d_attn_w[s] += d_xa[hs+j] * fwd_v[s][hs+j];
bwd_dv[s][hs+j] += d_xa[hs+j] * fwd_attn_w[t][h][s];
}
}
double dot_aw = 0;
for (int s = 0; s <= t; s++)
dot_aw += fwd_attn_w[t][h][s] * d_attn_w[s];
double d_attn_logits[BLOCK_SIZE];
for (int s = 0; s <= t; s++)
d_attn_logits[s] = fwd_attn_w[t][h][s] * (d_attn_w[s] - dot_aw);
for (int s = 0; s <= t; s++) {
for (int j = 0; j < HEAD_DIM; j++) {
d_q[t][hs+j] += d_attn_logits[s] * fwd_k[s][hs+j] * inv_sqrt_hd;
bwd_dk[s][hs+j] += d_attn_logits[s] * fwd_q[t][hs+j] * inv_sqrt_hd;
}
}
}
}
/* Phase 2 (forward): QKV projections → rmsnorm → embeddings */
for (int t = 0; t < n; t++) {
int tok = tokens[t];
int pos = t;
double d_xn1_t[N_EMBD];
memset(d_xn1_t, 0, sizeof(d_xn1_t));
for (int r = 0; r < N_EMBD; r++) {
for (int c = 0; c < N_EMBD; c++) {
G(OFF_WQ, r, c, N_EMBD) += d_q[t][r] * fwd_xn1[t][c];
d_xn1_t[c] += d_q[t][r] * params[OFF_WQ + r * N_EMBD + c];
}
}
for (int r = 0; r < N_EMBD; r++) {
for (int c = 0; c < N_EMBD; c++) {
G(OFF_WK, r, c, N_EMBD) += bwd_dk[t][r] * fwd_xn1[t][c];
d_xn1_t[c] += bwd_dk[t][r] * params[OFF_WK + r * N_EMBD + c];
}
}
for (int r = 0; r < N_EMBD; r++) {
for (int c = 0; c < N_EMBD; c++) {
G(OFF_WV, r, c, N_EMBD) += bwd_dv[t][r] * fwd_xn1[t][c];
d_xn1_t[c] += bwd_dv[t][r] * params[OFF_WV + r * N_EMBD + c];
}
}
double d_x0_t[N_EMBD];
{
double sc = fwd_xn1_sc[t];
double dot = 0;
for (int i = 0; i < N_EMBD; i++) dot += d_xn1_t[i] * fwd_x0[t][i];
double sc3 = sc * sc * sc;
double coeff = sc3 * dot / N_EMBD;
for (int i = 0; i < N_EMBD; i++)
d_x0_t[i] = sc * d_xn1_t[i] - fwd_x0[t][i] * coeff;
}
for (int i = 0; i < N_EMBD; i++)
d_x0_t[i] += bwd_d_xr1[t][i];
double d_emb_t[N_EMBD];
{
double sc = fwd_x0_scale[t];
double dot = 0;
for (int i = 0; i < N_EMBD; i++) dot += d_x0_t[i] * fwd_emb[t][i];
double sc3 = sc * sc * sc;
double coeff = sc3 * dot / N_EMBD;
for (int i = 0; i < N_EMBD; i++)
d_emb_t[i] = sc * d_x0_t[i] - fwd_emb[t][i] * coeff;
}
for (int i = 0; i < N_EMBD; i++) {
G(OFF_WTE, tok, i, N_EMBD) += d_emb_t[i];
G(OFF_WPE, pos, i, N_EMBD) += d_emb_t[i];
}
}
}
/* ── Inference forward ───────────────────────────────────────────── */
static double inf_k[BLOCK_SIZE][N_EMBD];
static double inf_v[BLOCK_SIZE][N_EMBD];
static void inference_forward(int tok, int pos, double *logits_out) {
double x[N_EMBD], tmp[N_EMBD];
for (int i = 0; i < N_EMBD; i++)
x[i] = W(OFF_WTE, tok, i, N_EMBD) + W(OFF_WPE, pos, i, N_EMBD);
do_rmsnorm(x, x, N_EMBD);
double xr[N_EMBD];
memcpy(xr, x, sizeof(xr));
do_rmsnorm(x, tmp, N_EMBD);
double q[N_EMBD], k[N_EMBD], v[N_EMBD];
do_linear(tmp, OFF_WQ, N_EMBD, N_EMBD, q);
do_linear(tmp, OFF_WK, N_EMBD, N_EMBD, k);
do_linear(tmp, OFF_WV, N_EMBD, N_EMBD, v);
memcpy(inf_k[pos], k, sizeof(k));
memcpy(inf_v[pos], v, sizeof(v));
double xa[N_EMBD];
double inv_sqrt_hd = 1.0 / sqrt((double)HEAD_DIM);
for (int h = 0; h < N_HEAD; h++) {
int hs = h * HEAD_DIM;
double attn_logits[BLOCK_SIZE];
for (int s = 0; s <= pos; s++) {
double dot = 0;
for (int j = 0; j < HEAD_DIM; j++)
dot += q[hs+j] * inf_k[s][hs+j];
attn_logits[s] = dot * inv_sqrt_hd;
}
double mx = attn_logits[0];
for (int s = 1; s <= pos; s++) if (attn_logits[s] > mx) mx = attn_logits[s];
double weights[BLOCK_SIZE], total = 0;
for (int s = 0; s <= pos; s++) {
weights[s] = exp(attn_logits[s] - mx);
total += weights[s];
}
for (int s = 0; s <= pos; s++) weights[s] /= total;
for (int j = 0; j < HEAD_DIM; j++) {
double acc = 0;
for (int s = 0; s <= pos; s++)
acc += weights[s] * inf_v[s][hs+j];
xa[hs+j] = acc;
}
}
double wo_out[N_EMBD];
do_linear(xa, OFF_WO, N_EMBD, N_EMBD, wo_out);
for (int i = 0; i < N_EMBD; i++)
x[i] = wo_out[i] + xr[i];
memcpy(xr, x, sizeof(xr));
do_rmsnorm(x, tmp, N_EMBD);
double h1[MLP_DIM];
do_linear(tmp, OFF_FC1, MLP_DIM, N_EMBD, h1);
for (int i = 0; i < MLP_DIM; i++) if (h1[i] < 0) h1[i] = 0;
double fc2[N_EMBD];
do_linear(h1, OFF_FC2, N_EMBD, MLP_DIM, fc2);
for (int i = 0; i < N_EMBD; i++)
x[i] = fc2[i] + xr[i];
do_linear(x, OFF_LMHEAD, VOCAB_SIZE, N_EMBD, logits_out);
}
/* ── Main ────────────────────────────────────────────────────────── */
int main(void) {
uint32_t seed_key[1] = {42};
mt_init_by_array(seed_key, 1);
load_data();
rng_shuffle(doc_idx, num_docs);
printf("num docs: %d\n", num_docs);
printf("vocab size: %d\n", VOCAB_SIZE);
for (int i = 0; i < NUM_PARAMS; i++)
params[i] = rng_gauss(0, 0.08);
printf("num params: %d\n", NUM_PARAMS);
memset(adam_m, 0, sizeof(adam_m));
memset(adam_v, 0, sizeof(adam_v));
double lr = 0.01, beta1 = 0.85, beta2 = 0.99, eps_adam = 1e-8;
int num_steps = 1000;
for (int step = 0; step < num_steps; step++) {
const char *doc = docs[doc_idx[step % num_docs]];
int doclen = (int)strlen(doc);
int tokens[BLOCK_SIZE + 2];
int copylen = doclen;
if (copylen > BLOCK_SIZE - 1) copylen = BLOCK_SIZE - 1;
tokens[0] = VOCAB_SIZE - 1;
for (int i = 0; i < copylen; i++)
tokens[i + 1] = doc[i] - 'a';
tokens[copylen + 1] = VOCAB_SIZE - 1;
int n = copylen + 1;
double loss = train_forward(tokens, n);
train_backward(tokens, n);
double lr_t = lr * (1.0 - (double)step / num_steps);
double bc1 = 1.0 - pow(beta1, step + 1);
double bc2 = 1.0 - pow(beta2, step + 1);
for (int i = 0; i < NUM_PARAMS; i++) {
adam_m[i] = beta1 * adam_m[i] + (1.0 - beta1) * grads[i];
adam_v[i] = beta2 * adam_v[i] + (1.0 - beta2) * grads[i] * grads[i];
double m_hat = adam_m[i] / bc1;
double v_hat = adam_v[i] / bc2;
params[i] -= lr_t * m_hat / (sqrt(v_hat) + eps_adam);
}
printf("step %4d / %4d | loss %.4f\n", step + 1, num_steps, loss);
}
double temperature = 0.5;
printf("--- inference (new, hallucinated names) ---\n");
for (int s = 0; s < 20; s++) {
int tok = VOCAB_SIZE - 1;
char sample[BLOCK_SIZE + 1];
int slen = 0;
for (int pos = 0; pos < BLOCK_SIZE; pos++) {
double logits[VOCAB_SIZE];
inference_forward(tok, pos, logits);
for (int i = 0; i < VOCAB_SIZE; i++) logits[i] /= temperature;
double probs[VOCAB_SIZE];
do_softmax(logits, probs, VOCAB_SIZE);
tok = rng_weighted_choice(probs, VOCAB_SIZE);
if (tok == VOCAB_SIZE - 1) break;
sample[slen++] = 'a' + tok;
}
sample[slen] = '\0';
printf("sample %2d: %s\n", s + 1, sample);
}
return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment