Nix remote-build auto-GC contention reproduction attempt

README.md

Remote-build auto-GC contention reproduction attempt

This NixOS VM test creates a client and a four-slot SSH builder, seeds 800 unreachable client store paths, forces the client's reported free space below min-free, and releases eight remote build outputs together.

Run it with:

nix build path:.#checks.x86_64-linux.default -L \
  --builders '' --min-free 0 --max-free 0

The final two options prevent the host Nix daemon's own automatic GC from interfering with the nested VM test.

Result

The test completed on unpatched Nix 2.34.8 in about 99 seconds. All eight outputs returned, so this attempt did not reproduce the production hang deterministically.

The production GDB capture showed an extra condition that this test did not reliably force: the remote build hook's auto-GC filled its JSON log pipe while the parent daemon entered a second synchronous auto-GC. The parent then stopped draining the hook while it waited for the GC lock held by that hook.

The VM test remains useful as a runnable stress harness and records the attempted reproduction. It should not be treated as regression coverage for the patch.

flake.lock

{
  "nodes": {
    "nixpkgs": {
      "locked": {
        "lastModified": 1785692966,
        "narHash": "sha256-vUfIeBEfpbAfZ5zjgIkYk7eHBeVfCYVjLbWnMkseYnk=",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "643809054d65fdd466a63e3155b8c498cb483c04",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "ref": "nixos-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "root": {
      "inputs": {
        "nixpkgs": "nixpkgs"
      }
    }
  },
  "root": "root",
  "version": 7
}

flake.nix

{
  description = "Reproduce remote-build auto-GC lock contention";

  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";

  outputs = { self, nixpkgs }:
    let
      system = "x86_64-linux";
      pkgs = nixpkgs.legacyPackages.${system};
    in {
      checks.${system}.default = pkgs.testers.runNixOSTest {
        name = "remote-build-auto-gc-contention";

        nodes.builder = { pkgs, ... }: {
          services.openssh.enable = true;
          virtualisation.writableStore = true;
          # Expose the synchronization files in /tmp to the test driver.
          nix.settings.sandbox = false;
        };

        nodes.client = { config, lib, pkgs, ... }: {
          virtualisation.writableStore = true;
          virtualisation.additionalPaths = [ pkgs.busybox ];
          nix.distributedBuilds = true;
          nix.settings = {
            max-jobs = 0;
            substituters = lib.mkForce [ ];
            min-free = 1000000;
            max-free = 2000000;
            min-free-check-interval = 0;
            experimental-features = [ "nix-command" "flakes" ];
          };
          nix.buildMachines = [{
            hostName = "builder";
            sshUser = "root";
            sshKey = "/root/.ssh/id_ed25519";
            system = system;
            maxJobs = 4;
          }];
          systemd.services.nix-daemon.path = [ pkgs.openssh ];
          systemd.services.nix-daemon.environment._NIX_TEST_FREE_SPACE_FILE = "/run/nix-fake-free";
          systemd.tmpfiles.rules = [ "f+ /run/nix-fake-free 0644 root root - 10000000" ];
          programs.ssh.extraConfig = ''
            Host builder
              StrictHostKeyChecking no
              UserKnownHostsFile /dev/null
          '';
        };

        testScript = { nodes, ... }: let
          expression = pkgs.writeText "remote-builds.nix" ''
            let
              busybox = builtins.storePath ${pkgs.busybox};
              mk = n: derivation {
                name = "remote-auto-gc-" + toString n;
                system = "${system}";
                PATH = "''${busybox}/bin";
                builder = "''${busybox}/bin/sh";
                args = [ "-c" "touch /tmp/remote-build-started-$name; while ! test -e /tmp/release-remote-builds; do sleep 0.1; done; mkdir $out; printf '%s\\n' \"$name\" > $out/result" ];
              };
            in map mk [ 1 2 3 4 5 6 7 8 ]
          '';
        in ''
          import shlex
          import subprocess
          import time

          start_all()
          client.wait_for_unit("nix-daemon.socket")
          builder.wait_for_unit("sshd.service")

          subprocess.run([
            "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", ""
          ], capture_output=True, check=True)
          client.succeed("install -d -m 700 /root/.ssh")
          client.copy_from_host("key", "/root/.ssh/id_ed25519")
          client.succeed("chmod 600 /root/.ssh/id_ed25519")
          builder.succeed("install -d -m 700 /root/.ssh")
          builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
          client.succeed("ssh builder true")

          # Seed enough dead paths for auto-GC to fill a build-hook log pipe.
          client.succeed("""
            mkdir /tmp/garbage
            for i in $(seq 1 800); do
              printf '%08d\n' "$i" > /tmp/garbage/item
              nix store add-file --name "auto-gc-garbage-$i" /tmp/garbage/item >/dev/null
            done
          """)

          drvs = client.succeed("nix-instantiate '${expression}'").split()
          command = "/run/current-system/sw/bin/nix-store -r " + " ".join(drvs) + " >/tmp/build.out 2>/tmp/build.err; echo $? >/tmp/build.status"
          client.succeed(
              "systemd-run --unit=repro-build --collect "
              "--setenv=PATH=/run/current-system/sw/bin:${pkgs.openssh}/bin "
              "/bin/sh -c " + shlex.quote(command)
          )
          # Give the first batch time to enter their builders. The release file
          # may appear before a slower builder reaches its polling loop.
          time.sleep(5)

          # Make every local store write request synchronous auto-GC, then let
          # several remote outputs arrive together.
          client.succeed("echo 1 > /run/nix-fake-free")
          builder.succeed("touch /tmp/release-remote-builds")

          time.sleep(10)
          status = client.execute("test -e /tmp/build.status")[0]
          if status != 0:
              client.succeed("ps -eo pid,ppid,stat,wchan:24,etime,cmd > /tmp/hung-processes")
              client.succeed("lslocks > /tmp/hung-locks")
              raise Exception("remote build remained blocked after outputs completed; auto-GC lock contention reproduced")

          print(client.succeed("cat /tmp/build.err; cat /tmp/build.status"))
          client.succeed("test $(cat /tmp/build.status) -eq 0")
          client.succeed("test $(wc -l < /tmp/build.out) -eq 8")
        '';
      };
    };
}