bpfcanary.go

// Command bpfcanary checks whether the kernel writes past the end of the
// truncated bpf_attr structures used by opencontainers/cgroups/devices.
//
// Each attr is placed at the start of a larger buffer whose tail is filled
// with a known pattern. After each bpf(2) call the tail is checked: any byte
// that changed means the kernel wrote beyond the size we passed, which is the
// suspected cause of the memory corruption seen in runc's CI.
//
// Needs root (CAP_BPF/CAP_NET_ADMIN) and a cgroup2 mount. Stdlib only, so it
// can be run with "go run bpfcanary.go".
package main

import (
	"encoding/binary"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"syscall"
	"unsafe"
)

// bpf(2) syscall numbers, per GOARCH.
var sysBPF = map[string]uintptr{
	"amd64": 321,
	"arm64": 280,
}

// enum bpf_cmd
const (
	cmdProgLoad      = 5
	cmdProgAttach    = 8
	cmdProgDetach    = 9
	cmdProgGetFdByID = 13
	cmdProgQuery     = 16
)

const (
	progTypeCgroupDevice = 15 // enum bpf_prog_type
	attachCgroupDevice   = 6  // enum bpf_attach_type
	flagAllowMulti       = 1 << 1
)

const (
	canaryLen  = 256
	canaryByte = 0xa5
)

// keepalive holds every buffer whose address is handed to the kernel, so
// nothing is collected while a call is in flight.
var keepalive []any

// call runs bpf(2) with attr, passing len(attr) as the size, and reports any
// write past the end of attr.
func call(name string, cmd uintptr, attr []byte) (uintptr, syscall.Errno) {
	buf := make([]byte, len(attr)+canaryLen)
	copy(buf, attr)
	for i := len(attr); i < len(buf); i++ {
		buf[i] = canaryByte
	}
	before := make([]byte, len(attr))
	copy(before, buf[:len(attr)])

	r1, _, errno := syscall.Syscall(sysBPF[runtime.GOARCH], cmd,
		uintptr(unsafe.Pointer(&buf[0])), uintptr(len(attr)))
	runtime.KeepAlive(buf)

	status := "ok"
	if errno != 0 {
		status = errno.Error()
	}
	fmt.Printf("%-18s size=%-3d ret=%-4d %s\n", name, len(attr), int(r1), status)

	// Write-back inside attr is expected for some commands; report it so we
	// can tell it apart from an overflow.
	for i := range before {
		if buf[i] != before[i] {
			fmt.Printf("    in-bounds write-back at offset %d: % x -> % x\n",
				i, before[i:], buf[i:len(attr)])
			break
		}
	}

	// Hand the kernel's write-back to the caller (e.g. query.prog_cnt).
	copy(attr, buf[:len(attr)])

	overflow := -1
	for i := len(attr); i < len(buf); i++ {
		if buf[i] != canaryByte {
			overflow = i
			break
		}
	}
	if overflow >= 0 {
		last := overflow
		for i := len(buf) - 1; i >= overflow; i-- {
			if buf[i] != canaryByte {
				last = i
				break
			}
		}
		fmt.Printf("    *** OVERFLOW: kernel wrote at offsets %d..%d (attr is %d bytes)\n",
			overflow, last, len(attr))
		fmt.Printf("    *** tail: % x\n", buf[len(attr):last+1+8])
	}
	return r1, errno
}

func u32(b []byte, off int, v uint32) { binary.NativeEndian.PutUint32(b[off:], v) }
func u64(b []byte, off int, v uint64) { binary.NativeEndian.PutUint64(b[off:], v) }
func ptr(v any, p unsafe.Pointer) uint64 {
	keepalive = append(keepalive, v)
	return uint64(uintptr(p))
}

func main() {
	if _, ok := sysBPF[runtime.GOARCH]; !ok {
		fmt.Fprintf(os.Stderr, "unsupported GOARCH %s\n", runtime.GOARCH)
		os.Exit(1)
	}
	if os.Geteuid() != 0 {
		fmt.Fprintln(os.Stderr, "must run as root")
		os.Exit(1)
	}

	dir := filepath.Join("/sys/fs/cgroup", "bpf-canary-probe")
	if err := os.Mkdir(dir, 0o755); err != nil && !os.IsExist(err) {
		fmt.Fprintf(os.Stderr, "mkdir %s: %v (is this a cgroup2 mount?)\n", dir, err)
		os.Exit(1)
	}
	defer os.Remove(dir)
	dirFile, err := os.Open(dir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "open %s: %v\n", dir, err)
		os.Exit(1)
	}
	defer dirFile.Close()
	dirFd := uint32(dirFile.Fd())

	// BPF_PROG_LOAD: "mov64 r0, 0; exit", the same trivial program the
	// BPF_F_REPLACE probe in devices/ebpf_linux.go loads.
	insns := []byte{
		0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r0, 0
		0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // exit
	}
	license := []byte("MIT\x00")

	load := make([]byte, 40)
	u32(load, 0, progTypeCgroupDevice)
	u32(load, 4, uint32(len(insns)/8))
	u64(load, 8, ptr(insns, unsafe.Pointer(&insns[0])))
	u64(load, 16, ptr(license, unsafe.Pointer(&license[0])))
	// logLevel/logSize/logBuf left zero.
	r1, errno := call("BPF_PROG_LOAD", cmdProgLoad, load)
	if errno != 0 {
		fmt.Fprintln(os.Stderr, "prog load failed, cannot continue")
		os.Exit(1)
	}
	progFd := uint32(r1)
	defer syscall.Close(int(progFd))

	// BPF_PROG_ATTACH
	attach := make([]byte, 20)
	u32(attach, 0, dirFd)
	u32(attach, 4, progFd)
	u32(attach, 8, attachCgroupDevice)
	u32(attach, 12, flagAllowMulti)
	call("BPF_PROG_ATTACH", cmdProgAttach, attach)

	// BPF_PROG_QUERY
	progIds := make([]uint32, 64)
	query := make([]byte, 32)
	u32(query, 0, dirFd)
	u32(query, 4, attachCgroupDevice)
	u64(query, 16, ptr(progIds, unsafe.Pointer(&progIds[0])))
	u32(query, 24, uint32(len(progIds)))
	call("BPF_PROG_QUERY", cmdProgQuery, query)
	nIDs := binary.NativeEndian.Uint32(query[24:])
	fmt.Printf("    query returned %d prog id(s)\n", nIDs)

	// BPF_PROG_GET_FD_BY_ID -- the 4-byte attr, the prime suspect.
	if nIDs > 0 {
		getfd := make([]byte, 4)
		u32(getfd, 0, progIds[0])
		r1, errno := call("BPF_PROG_GET_FD_BY_ID", cmdProgGetFdByID, getfd)
		if errno == 0 {
			syscall.Close(int(r1))
		}
	}

	// BPF_PROG_DETACH
	detach := make([]byte, 12)
	u32(detach, 0, dirFd)
	u32(detach, 4, progFd)
	u32(detach, 8, attachCgroupDevice)
	call("BPF_PROG_DETACH", cmdProgDetach, detach)

	fmt.Println("\ndone; any line marked *** OVERFLOW identifies the culprit")
}