karpathy

LLM Wiki

A pattern for building personal knowledge bases using LLMs.

This is an idea file, it is designed to be copy pasted to your own LLM Agent (e.g. OpenAI Codex, Claude Code, OpenCode / Pi, or etc.). Its goal is to communicate the high level idea, but your agent will build out the specifics in collaboration with you.

The core idea

Most people's experience with LLMs and documents looks like RAG: you upload a collection of files, the LLM retrieves relevant chunks at query time, and generates an answer. This works, but the LLM is rediscovering knowledge from scratch on every question. There's no accumulation. Ask a subtle question that requires synthesizing five documents, and the LLM has to find and piece together the relevant fragments every time. Nothing is built up. NotebookLM, ChatGPT file uploads, and most RAG systems work this way.

karpathy

"""
The most atomic way to train and run inference for a GPT in pure, dependency-free Python.
This file is the complete algorithm.
Everything else is just efficiency.

@karpathy
"""

import os       # os.path.exists
import math     # math.log, math.exp

araffin

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from gymnasium import spaces
from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.vec_env import VecEnvWrapper

sns.set_theme()

willccbb

# train_grpo.py
#
# See https://github.com/willccbb/verifiers for ongoing developments
#
"""
citation:

@misc{brown2025grpodemo,
  title={Granular Format Rewards for Eliciting Mathematical Reasoning Capabilities in Small Language Models},
  author={Brown, William},

karpathy

# -----------------------------------------------------------------------------
# AI-powered Git Commit Function
# Copy paste this gist into your ~/.bashrc or ~/.zshrc to gain the `gcm` command. It:
# 1) gets the current staged changed diff
# 2) sends them to an LLM to write the git commit message
# 3) allows you to easily accept, edit, regenerate, cancel
# But - just read and edit the code however you like
# the `llm` CLI util is awesome, can get it here: https://llm.datasette.io/en/stable/

gcm() {

awni

hamelsmu

from fastapi import Request, HTTPException
from pydantic import BaseModel, BaseModel, HttpUrl
from modal import Secret, App, web_endpoint, Image
from typing import Optional, List
from example import proposal
import os

app = App(name="circleback", image=Image.debian_slim().pip_install("openai", "pydantic", "fastapi"))

class Attendee(BaseModel):

shreyshahi

# Code inspired from https://gist.github.com/karpathy/00103b0037c5aaea32fe1da1af553355
# slerp function is entirely lifted from the above gist.

import torch
from diffusers import DiffusionPipeline
import numpy as np


def interpolate(v1, v2, step, total_steps):
    alpha = step / (total_steps - 1)

thomwolf

""" To use: install LLM studio (or Ollama), clone OpenVoice, run this script in the OpenVoice directory
    git clone https://github.com/myshell-ai/OpenVoice
    cd OpenVoice
    git clone https://huggingface.co/myshell-ai/OpenVoice
    cp -r OpenVoice/* .
    pip install whisper pynput pyaudio
"""

from openai import OpenAI
import time

atiorh

# !pip install transformers sentencepiece

import torch
import torch.nn as nn
torch.set_grad_enabled(False)

from transformers import AutoTokenizer, AutoModelForCausalLM
from transformers.activations import ReLUSquaredActivation

from collections import defaultdict, OrderedDict