Jörg Thalheim Mic92

Benchmark: parallel `addMultipleToStore` for `nix copy`

Measures the performance impact of parallelizing LocalStore::addMultipleToStore (writing store paths to disk in parallel via a thread pool, then registering in a single SQLite transaction) vs the base sequential implementation.

Test closure: Firefox — ~372 paths, ~1.6GB total.

Setup

hyperfine --show-output 'rm -R ~/.cache/nix/binary-cache-*.sqlite*; rm -rf /tmp/cache; nix copy --from "{url}" --to "file:///tmp/cache?compression=none" /nix/store/jlkypcf54nrh4n6r0l62ryx93z752hb2-firefox-132.0' -L url https://nixos.tvix.store,https://cache.nixos.org/
Benchmark 1: rm -R ~/.cache/nix/binary-cache-*.sqlite*; rm -rf /tmp/cache; nix copy --from "https://nixos.tvix.store" --to "file:///tmp/cache?compression=none" /nix/store/jlkypcf54nrh4n6r0l62ryx93z752hb2-firefox-132.0
  Time (mean ± σ):      5.090 s ±  0.113 s    [User: 4.721 s, System: 2.555 s]
  Range (min … max):    4.942 s …  5.332 s    10 runs

Benchmark 2: rm -R ~/.cache/nix/binary-cache-*.sqlite*; rm -rf /tmp/cache; nix copy --from "https://cache.nixos.org/" --to "file:///tmp/cache?compression=none" /nix/store/jlkypcf54nrh4n6r0l62ryx93z752hb2-firefox-132.0
  Time (mean ± σ):      6.729 s ±  0.148 s    [User: 15.829 s, System: 2.904 s]
  Range (min … max):    6.560 s …  7.049 s    10 runs

	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="UTF-8">
	<meta name="viewport" content="width=device-width, initial-scale=1.0">
	<title>computeClosure async — State Machine & Sequence Diagrams</title>
	<style>
	* { margin: 0; padding: 0; box-sizing: border-box; }
	body { font-family: 'Segoe UI', system-ui, -apple-system, sans-serif; background: #0d1117; color: #c9d1d9; padding: 32px 48px; }
	h1 { color: #58a6ff; font-size: 1.4em; margin-bottom: 4px; }

	{
	"nodes": {
	"blueprint": {
	"inputs": {
	"nixpkgs": [
	"llm-agents",
	"nixpkgs"
	],
	"systems": "systems"
	},

	# Declarative Authelia users with secret file references
	# Secrets are resolved at service start, never stored in the Nix store
	{
	config,
	lib,
	pkgs,
	...
	}:
	let
	cfg = config.services.authelia.instances.main.declarativeUsers;

	parallel_map() {
	local func="$1"

	local num_cores
	num_cores=$(nproc 2>/dev/null \|\| echo 4)
	local pipe_buf_size
	pipe_buf_size=$(getconf PIPE_BUF / 2>/dev/null \|\| echo 4096)
	local tmpdir
	tmpdir=$(mktemp -d)
	trap 'rm -rf "$tmpdir"' EXIT

	From 3ceb739b696ac712dddfc6acf1f88e492d6f07d3 Mon Sep 17 00:00:00 2001
	From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
	Date: Sat, 20 Jul 2024 10:23:57 +0200
	Subject: [PATCH] detect systemd by name rather than relying on
	SYSTEMD_BINARY_PATH
	MIME-Version: 1.0
	Content-Type: text/plain; charset=UTF-8
	Content-Transfer-Encoding: 8bit

	This allows us in NixOS to run systemd binaries from the nix store

	#!/bin/bash

	set -eux -o pipefail

	# Create mount point if it doesn't exist
	mkdir -p /run/testmount

	# Clean up on exit
	trap 'umount /run/testmount; rmdir /run/testmount' EXIT

	{ pkgs, config, ... }:
	{
	nixpkgs.config.allowUnfree = true;
	nixpkgs.config.nvidia.acceptLicense = true;
	# enable the nvidia driver
	services.xserver.videoDrivers = [ "nvidia" ];
	hardware.opengl.enable = true;
	hardware.nvidia.datacenter.enable = true;
	hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.dc_535;

Jörg Thalheim Mic92

Benchmark: parallel addMultipleToStore for nix copy

Setup

Benchmark: parallel `addMultipleToStore` for `nix copy`