Utils to migrate ZFS snapshots to BTRFS

convert.py

#!/usr/bin/env python3
__author__ = "Vyacheslav Napadovsky, 2026"
__comment__ = "Convert ZFS to BTRFS script"
from __future__ import annotations

import argparse
import atexit
import os
import re
import shlex
import shutil
import stat
import subprocess
import sys
import tempfile
import socket
from pathlib import Path

def run_cmd(argv, *, capture=False, check=True, text=True):
    print(f"Running: {shlex.join(argv)}");
    try:
        if capture:
            cp = subprocess.run(argv, check=check, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=text)
            return cp.stdout
        else:
            subprocess.run(argv, check=check)
            return None
    except subprocess.CalledProcessError as e:
        stderr = ""
        if getattr(e, "stderr", None):
            stderr = e.stderr.strip()
        raise RuntimeError(f"command failed: {shlex.join(argv)}" + (f"\n{stderr}" if stderr else ""))

def umount_if_mounted(path: Path) -> None:
    if subprocess.run(["mountpoint", "-q", str(path)]).returncode == 0:
        run_cmd(["umount", str(path)])

def mount_zfs_ro(ztarget: str, mnt: Path) -> None:
    run_cmd(["mount", "-t", "zfs", "-o", "ro", ztarget, str(mnt)])



CHUNK_SIZE = 128 * 1024  # 128 kiB

def read_exact(f, chunk):
    result = []
    while chunk > 0:
        r = f.read(chunk)
        if not r:
            break
        chunk -= len(r)
        result.append(r)
    return b''.join(result)

def sync_file(src: Path, dst: Path) -> None:
    if dst.exists():
        changed = False
        if dst.is_symlink() or not dst.is_file():
            remove_path(dst)
        #dst_fd = os.open(dst, os.O_CREAT | os.O_RDWR, 0o644)
        #with src.open("rb") as sf, os.fdopen(dst_fd, "r+b") as df:
        with src.open("rb") as sf, dst.open("r+b") as df:
            while True:
                sblk = read_exact(sf, CHUNK_SIZE)
                if not sblk:
                    break
                dblk = read_exact(df, len(sblk)) # may be shorter near EOF

                if dblk != sblk:
                    if len(dblk) > 0:
                        df.seek(-len(dblk), 1)
                    df.write(sblk)
                    changed = True
            df.truncate()
    else:
        raise RuntimeError("logic error")
        #changed = True
        #shutil.copyfile(src, dst, follow_symlinks=False)

    # Always update metadata to match source
    shutil.copystat(src, dst, follow_symlinks=False)



def path_kind(p: Path) -> str:
    st = p.lstat()
    m = st.st_mode
    if stat.S_ISLNK(m):
        return "symlink"
    if stat.S_ISDIR(m):
        return "dir"
    if stat.S_ISREG(m):
        return "file"
    if stat.S_ISFIFO(m):
        return "fifo"
    if stat.S_ISCHR(m):
        return "char"
    if stat.S_ISBLK(m):
        return "block"
    if stat.S_ISSOCK(m):
        return "socket"
    return "unknown"

def remove_path(path: Path) -> None:
    try:
        k = path_kind(path)
    except FileNotFoundError:
        return

    if k in {"file", "symlink", "fifo", "char", "block", "socket", "unknown"}:
        path.unlink()
    elif k == "dir":
        shutil.rmtree(path)
    else:
        path.unlink()

def ensure_dir(path: Path) -> None:
    path.mkdir(parents=True, exist_ok=True)

def copy_symlink(src: Path, dst: Path) -> None:
    target = os.readlink(src)
    if dst.exists() or dst.is_symlink():
        if dst.is_symlink() and os.readlink(dst) == target:
            return
        remove_path(dst)
    os.symlink(target, dst)

def recreate_fifo(src: Path, dst: Path) -> None:
    src_st = src.lstat()
    if dst.exists():
        try:
            if path_kind(dst) != "fifo":
                remove_path(dst)
        except FileNotFoundError:
            pass
    if not dst.exists():
        dst.parent.mkdir(parents=True, exist_ok=True)
        os.mkfifo(dst, mode=stat.S_IMODE(src_st.st_mode))
    apply_special_metadata(src_st, dst)

def recreate_device(src: Path, dst: Path) -> None:
    src_st = src.lstat()
    src_mode = src_st.st_mode
    if not hasattr(os, "mknod"):
        raise RuntimeError(f"os.mknod not available on this platform; cannot recreate: {src}")
    if dst.exists():
        try:
            dk = path_kind(dst)
            desired = "char" if stat.S_ISCHR(src_mode) else "block"
            if dk != desired:
                remove_path(dst)
        except FileNotFoundError:
            pass
    if not dst.exists():
        dst.parent.mkdir(parents=True, exist_ok=True)
        # Keep the file type bits from src_mode; permissions in low bits.
        os.mknod(dst, mode=src_mode, device=src_st.st_rdev)
    apply_special_metadata(src_st, dst)

def recreate_unix_socket(src: Path, dst: Path) -> None:
    src_st = src.lstat()
    if dst.exists():
        try:
            if path_kind(dst) != "socket":
                remove_path(dst)
        except FileNotFoundError:
            pass
    if not dst.exists():
        dst.parent.mkdir(parents=True, exist_ok=True)
        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        try:
            s.bind(str(dst))
        finally:
            s.close()
    apply_special_metadata(src_st, dst)

def sync_special(src: Path, dst: Path) -> None:
    k = path_kind(src)
    if k == "fifo":
        recreate_fifo(src, dst)
    elif k in {"char", "block"}:
        recreate_device(src, dst)
    elif k == "socket":
        if os.name == "nt":
            raise RuntimeError(f"UNIX domain sockets not supported on this platform: {src}")
        recreate_unix_socket(src, dst)
    else:
        raise RuntimeError(f"Unsupported special file type '{k}': {src}")



def sync_tree(src_root: Path, dst_root: Path, delete: bool = True) -> None:
    if not src_root.exists() or not src_root.is_dir():
        raise RuntimeError(f"Source must be an existing directory: {src_root}")

    ensure_dir(dst_root)

    with os.scandir(src_root) as it:
        for entry in it:
            src_path = Path(entry.path)
            dst_path = dst_root / entry.name

            k = path_kind(src_path)
            if k == "symlink":
                #copy_symlink(src_path, dst_path)
                pass
            elif k == "dir":
                if dst_path.exists() and path_kind(dst_path) != "dir":
                    remove_path(dst_path)
                ensure_dir(dst_path)
                sync_tree(src_path, dst_path, delete=delete)
                shutil.copystat(src_path, dst_path, follow_symlinks=False)
            elif k == "file":
                sync_file(src_path, dst_path)
            elif k in {"fifo", "char", "block", "socket"}:
                #sync_special(src_path, dst_path)
                pass
            else:
                raise RuntimeError(f"SKIP unknown special: {src_path}")

_OCT_RE_B = re.compile(br"\\0([0-7]{1,3})")
def octal_escapes_to_bytes(data: bytes) -> bytes:
    def repl(m: re.Match[bytes]) -> bytes:
        return bytes([int(m.group(1), 8)])
    return _OCT_RE_B.sub(repl, data)

def apply_renames_from_zfsdiff(from_snap: str, to_snap: str, dest_root: Path, src_mount: Path) -> None:
    """
    Reads: zfs diff -H -h from to
    Only handles rename operations (op == 'R'), and replays them inside dest_root.
    """
    renamed = 0
    out = run_cmd(["zfs", "diff", "-H", from_snap, to_snap], capture=True) or ""
    for line in out.splitlines():
        # Format: op \t p1 \t p2 (for rename)
        parts = line.split("\t")
        if len(parts) < 3:
            continue
        op, p1, p2 = parts[0], parts[1], parts[2]
        if op != "R":
            continue

        p1 = octal_escapes_to_bytes(p1.encode()).decode("utf-8", "surrogateescape")
        p2 = octal_escapes_to_bytes(p1.encode()).decode("utf-8", "surrogateescape")

        # Strip "$SRC_MOUNT/" prefix like the bash script.
        src_prefix = str(src_mount).rstrip("/") + "/"
        if p1.startswith(src_prefix):
            p1_rel = p1[len(src_prefix) :]
        else:
            p1_rel = p1.lstrip(b"/")

        if p2.startswith(src_prefix):
            p2_rel = p2[len(src_prefix) :]
        else:
            p2_rel = p2.lstrip("/")

        src = dest_root / p1_rel
        dst = dest_root / p2_rel
        if True: #src.exists() and not dst.exists():
            dst.parent.mkdir(parents=True, exist_ok=True)
            # mv -T semantics: replace destination path exactly (no treating as dir)
            src.replace(dst)
            renamed += 1
    print(f"renamed: {renamed} files & dirs")


def main() -> None:
    p = argparse.ArgumentParser()
    p.add_argument("--dataset", required=True)
    p.add_argument("--dest-parent", required=True)
    p.add_argument("--name", required=True)
    p.add_argument("--src-mount", default="")
    args = p.parse_args()

    if os.geteuid() != 0:
        die("run as root")

    dest_parent = Path(args.dest_parent)
    if not dest_parent.is_dir():
        die(f"--dest-parent does not exist: {dest_parent}")

    name = args.name
    dataset = args.dataset
    dest_subvol = Path(str(dest_parent).rstrip("/")) / name
    dest_snapdir = Path(str(dest_parent).rstrip("/")) / f"{name}"
    work_base = Path(str(dest_parent).rstrip("/")) / f".zfs2btrfs-work-{name}"
    work = Path(str(work_base) + ".work")
    tmp_mnt = Path(str(dest_parent).rstrip("/")) / f".zfs2btrfs-mnt-{name}"

    def cleanup():
        try:
            umount_if_mounted(tmp_mnt)
        finally:
            try:
                tmp_mnt.rmdir()
            except OSError:
                pass
    atexit.register(cleanup)

    if work.exists():
        subprocess.run(["btrfs", "subvolume", "delete", str(work)])

    dest_snapdir.mkdir(parents=True, exist_ok=True)

    src_mount = args.src_mount
    if not src_mount:
        src_mount = (run_cmd(["zfs", "get", "-H", "-o", "value", "mountpoint", dataset], capture=True) or "").strip()
    if src_mount in ("legacy", "-"):
        raise (f"ZFS mountpoint is '{src_mount}'; provide --src-mount /path/to/mount")

    src_mount_path = Path(src_mount)
    if not src_mount_path.is_dir():
        raise (f"ZFS mountpoint directory not found: {src_mount_path}")

    snap_out = run_cmd(["zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-s", "creation", "-d", "1", dataset], capture=True)
    snapshots = [line.strip() for line in snap_out.splitlines() if line.strip()]

    tmp_mnt.mkdir(parents=True, exist_ok=True)

    prev_snap = ""
    run_cmd(["btrfs", "subvolume", "create", "--", str(work)])
    
    for snap in snapshots:
        umount_if_mounted(tmp_mnt)

        snap_short = snap.split("@", 1)[1] if "@" in snap else snap
        btrfs_snap_path = dest_snapdir / snap_short

        if btrfs_snap_path.exists():
            run_cmd(["btrfs", "subvolume", "delete", "--", str(work)])
            run_cmd(["btrfs", "subvolume", "snapshot", "--", str(btrfs_snap_path), str(work)])
        else:
            mount_zfs_ro(snap, tmp_mnt)
            if prev_snap:
                pass
                apply_renames_from_zfsdiff(prev_snap, snap, work, src_mount_path)
            run_cmd([
                "rsync", "-aAXUHx", "--inplace", "--no-whole-file",
                "--append", "--numeric-ids", "--no-i-r", "--delete", "--delete-after",
                "--info=progress2", "--", f"{str(tmp_mnt)}/", f"{str(work)}/",
            ])
            if prev_snap:
                sync_tree(tmp_mnt, work)
            run_cmd(["btrfs", "subvolume", "snapshot", "-r", "--", str(work), str(btrfs_snap_path)])
        prev_snap = snap


if __name__ == "__main__":
    main()

tidy.py

#!/usr/bin/env python3
__author__ = "Vyacheslav Napadovsky, 2026"
__comment__ = "Utility to remove content duplicates"
from __future__ import annotations

import argparse
import collections
import hashlib
import os
import stat
from pathlib import Path

PREFIX_SIZE =   1 * 1024**2   # 1 MiB
CHUNK_SIZE  = 256 * 1024**2   # 256 MiB

def read_exact(f, chunk):
    result = []
    while chunk > 0:
        r = f.read(chunk)
        if not r:
            break
        chunk -= len(r)
        result.append(r)
    return b''.join(result)

def is_equal(src: Path, dst: Path) -> None:
    if src.stat().st_size != dst.stat().st_size:
        return False
    with src.open("rb") as sf, dst.open("rb") as df:
        while True:
            sblk = read_exact(sf, CHUNK_SIZE)
            if not sblk:
                return True
            dblk = read_exact(df, len(sblk)) # may be shorter near EOF

            if dblk != sblk:
                return False

def path_kind(p: Path) -> str:
    st = p.lstat()
    m = st.st_mode
    if stat.S_ISLNK(m):
        return "symlink"
    if stat.S_ISDIR(m):
        return "dir"
    if stat.S_ISREG(m):
        return "file"
    if stat.S_ISFIFO(m):
        return "fifo"
    if stat.S_ISCHR(m):
        return "char"
    if stat.S_ISBLK(m):
        return "block"
    if stat.S_ISSOCK(m):
        return "socket"
    return "unknown"

def remove_path(path: Path) -> None:
    try:
        k = path_kind(path)
    except FileNotFoundError:
        return

    if k in {"file", "symlink", "fifo", "char", "block", "socket", "unknown"}:
        path.unlink()
    elif k == "dir":
        shutil.rmtree(path)
    else:
        path.unlink()

class MyFile:
    def __init__(self, p: Path):
        self.path = p
        self.size = p.stat().st_size
        with p.open("rb") as f:
          block = read_exact(f, PREFIX_SIZE)
        h = hashlib.sha256()
        h.update(block)
        self.hash = h.hexdigest()

    def is_same(self, f: MyFile):
        if self.size != f.size:
            return False
        if self.hash != f.hash:
            return False
        return is_equal(self.path, f.path)

def get_prefixes(root: Path):
    result = []
    with os.scandir(root) as it:
        for entry in it:
            path = Path(entry.path)
            k = path_kind(path)
            if k == "dir":
                result.extend(get_prefixes(path))
            elif k == "file":
                result.append(MyFile(path))
    return result

def cleanup(reference_map, work: Path):
    with os.scandir(work) as it:
        for entry in it:
            path = Path(entry.path)
            k = path_kind(path)
            if k == "dir":
                cleanup(reference_map, path)
                try:
                    os.rmdir(path)
                except OSError:
                    pass
            elif k == "file":
                f = MyFile(path)
                alts = reference_map.get(f.hash, None)
                if alts:
                    for alt in alts:
                        if alt.is_same(f):
                            f.path.unlink()
                            break

def main() -> None:
    p = argparse.ArgumentParser()
    p.add_argument("reference")
    p.add_argument("work")
    args = p.parse_args()

    ref = Path(args.reference)
    work = Path(args.work)
    if not ref.exists() or not ref.is_dir():
        raise RuntimeError(f"Reference must be an existing directory: {ref}") 
    if not work.exists() or not work.is_dir():
        raise RuntimeError(f"Work must be an existing directory: {work}") 

    reference_map = collections.defaultdict(list)
    for f in get_prefixes(ref):
        reference_map[f.hash].append(f)
    cleanup(reference_map, work)

if __name__ == "__main__":
    main()

tidy.sh

#!/bin/bash

# Utility to remove dedupes from BTRFS after conversion from ZFS
# Vyacheslav Napadovsky, 2026

set -euo pipefail

if [[ $EUID -ne 0 ]]; then
   echo "This script must be run as root"
   exit 1
fi

scripts="$(dirname "$0")"
cd test
last=( [0-9][0-9][0-9][0-9][0-9][0-9] )
cd ..
last="${last[-1]}"
input=( [0-9][0-9][0-9][0-9][0-9][0-9] )

for idx in "${!input[@]}"; do
  next="${input[$idx]}"
  if (( next <= last )); then
    continue;
  fi
#  echo "$scripts"/tidy.py "$next/" "test/$last/"
#  "$scripts"/tidy.py "$next/" "test/$last/"
  echo "$scripts"/tidy.py "$next/" "test/"
  "$scripts"/tidy.py "$next/" "test/"
  if (( idx != ${#input[@]} - 1)); then
    echo btrfs subvol snap -- "$next" test/"$next"
    btrfs subvol snap -- "$next" test/"$next"
  fi
  last="$next"
done