Rust CI Scripts

bark.bat

@powershell ./bark.ps1 %*

bark.ps1

$dir = "$PSScriptRoot"

if (!(test-path "$dir/target")) {
    new-item -ItemType Directory -Force -Path "$dir/target"
}

# Compile our "script"
if ($args[0] -eq "-f" -or !(test-path "$dir/target/bark")) {
    echo "Compiling bark!"

    rustc -g -o "$dir/target/bark.exe" "$dir/.ci/scripts/main.rs"

    if ($LASTEXITCODE -ne 0) {
        throw "failed to compile bark"
    }

    if ($args[0] -eq "-f") {
        # Remove the force flag
        $null, $args = $args
    }
}

&"$dir/target/bark.exe" $args

if ($LASTEXITCODE -ne 0) {
    throw "failed to run bark"
}

bark.sh

#!/bin/bash
set -eu

dir=$(dirname "$0")

if ! [ -d "${dir}/target" ]; then
    mkdir "${dir}/target"
fi

# Compile our "script" if the binary doesn't already exist
if [ "${1-}" == "-f" ] || ! [ -f "${dir}/target/bark" ]; then
    rustc -g -o "${dir}/target/bark" "${dir}/.ci/scripts/main.rs"
    
    if [ "${1-}" == "-f" ]; then
        # remove the force flag when sending the arguments to bark
        shift
    fi
fi

"${dir}/target/bark" "${@}"

main.rs

pub use std::{
    env,
    ffi::OsStr,
    fs,
    io::Write,
    process::{Command, Stdio},
};

mod buildkite;
mod cargo;
mod ctx;
mod modules;
mod sccache;
mod top;

pub use ctx::*;

const DEBUG_SCRIPT: bool = true;
const DEBUG_SCCACHE: bool = false;

fn main() {
    let is_ci = env::var("CI").is_ok() || env::var("BUILDKITE_COMMIT").is_ok();
    let job_id = env::var("BUILDKITE_JOB_ID").ok();

    let mut ctx = Ctx {
        channel: None,
        mode: None,
        host_os: ctx::get_host_os(),
        is_ci,
        with_sccache: is_ci,
        skip_build: false,
        job_id,
    };

    let mut subcommand = None;
    let mut args = Vec::new();

    for arg in env::args().skip(1) {
        if arg.starts_with('+') {
            println!("using channel '{}'", &arg[1..]);
            ctx.channel = Some(arg);
            continue;
        }

        if arg.starts_with("-") {
            if arg.starts_with("--") {
                match &arg[2..] {
                    "dev" => {
                        println!("using (dev)elopment mode");
                        ctx.mode = Some(Mode::Debug);
                    }
                    "prod" => {
                        println!("using prod (release) mode");
                        ctx.mode = Some(Mode::Release);
                    }
                    "sccache" => {
                        ctx.with_sccache = true;
                    }
                    "no-sccache" => {
                        ctx.with_sccache = false;
                    }
                    "skip-build" => {
                        ctx.skip_build = true;
                    }
                    unknown => {
                        if subcommand.is_none() {
                            eprintln!("ignored unknown flag '{}'", unknown);
                        } else {
                            args.push(arg);
                        }
                    }
                }
            } else {
                if subcommand.is_none() {
                    eprintln!("ignored unknown flag '{}'", arg);
                } else {
                    args.push(arg);
                }
            }

            continue;
        }

        match subcommand {
            None => {
                subcommand = Some(arg);
            }
            Some(_) => {
                args.push(arg);
            }
        }
    }

    let subcommand = subcommand.expect("no subcommand specified");

    install_native_deps(&ctx);

    println!("running subcommand {}", subcommand);

    fn to_opt(args: &[String]) -> Option<&str> {
        args.get(0).map(|s| s.as_str())
    }

    match subcommand.as_str() {
        "build" => top::build(&ctx, to_opt(&args)),
        "clippy" => top::clippy(&ctx, args),
        "download" => top::download(&ctx, args),
        "generate" => top::generate(&ctx, to_opt(&args)),
        "mirror" => top::mirror(&ctx, to_opt(&args)),
        "publish" => {
            // Just as with eg cargo install, we invert the usual default of debug
            // and instead default to release unless it's explicitly specified
            if ctx.mode.is_none() {
                println!("publish defaulting to release profile");
                ctx.mode = Some(Mode::Release);
            }

            top::publish(&ctx, args)
        }
        "sign-installer" => ark::sign_installer(&ctx, args),
        "strip" => top::strip(&ctx, args),
        "test" => top::test(&ctx, args),
        "upload" => top::upload(&ctx, args),
        "validate" => top::validate(&ctx, args),
        "watch" => top::watch(&ctx, args),
        unknown => panic!("unknown subcommand '{}' provided", unknown),
    }
}

fn print_version(ctx: &Ctx) {
    println!("--- :rust: version");

    let mut cmd = Command::new("rustc");

    if let Some(ref channel) = ctx.channel {
        cmd.arg(channel.as_str());
    }

    cmd.arg("--version");
    ctx.exec(cmd).expect("failed to get rustc version");
}

fn install_native_deps(ctx: &Ctx) {
    if ctx.host_os == HostOs::Mac {
        println!("--- Installing prerequisites");
        let mut cmd = Command::new("brew");
        cmd.args(&["install", "cmake", "python", "--display-times"]);
        // brew "fails" if it's already up to date, just ignore it
        let _ = ctx.exec(cmd);
    }
}

top.rs

// These are the top level commands that can be executed as subcommands of the
// same name via bark

fn which<'a>(args: &'a [String], def: &'a str) -> &'a str {
    args.get(0).map(|s| s.as_str()).unwrap_or(def)
}

pub fn validate(ctx: &Ctx, args: Vec<String>) {
    let which = which(&args, "all");

    cargo::validate_fmt(ctx);

    if which == "all" {
        cargo::fetch(ctx, "ark");
        cargo::fetch(ctx, "ark-api");
        cargo::fetch(ctx, "modules");

        cargo::deny(ctx, "ark");
        cargo::deny(ctx, "modules");

        cargo::clippy(ctx, "ark");
        cargo::clippy(ctx, "ark-api");
        cargo::clippy(ctx, "modules");
    } else {
        cargo::fetch(ctx, which);
        cargo::deny(ctx, which);
        cargo::clippy(ctx, which);
    }
}

pub fn clippy(ctx: &Ctx, args: Vec<String>) {
    let which = which(&args, "all");
    if which == "all" {
        cargo::fetch(ctx, "ark");
        cargo::fetch(ctx, "ark-api");
        cargo::fetch(ctx, "modules");

        cargo::clippy(ctx, "ark");
        cargo::clippy(ctx, "ark-api");
        cargo::clippy(ctx, "modules");
    } else {
        cargo::fetch(ctx, which);
        cargo::clippy(ctx, which);
    }
}

pub fn test(ctx: &Ctx, args: Vec<String>) {
    let which = which(&args, "ark");

    if !ctx.skip_build {
        // We split out the building and actually running the tests
        // into a separate step so we can more easily see timings, esp in CI
        match which {
            "ark" => {
                build(ctx, Some("ark-test"));
            }
            "modules" => {
                build(ctx, Some("ark-client"));
                build(ctx, Some("modules"));
            }
            unknown => panic!("unknown test target '{}'", unknown),
        }
    }

    println!("--- Running tests :suspect:");
    match which {
        "ark" => {
            ctx.cargo(&["test", "--all", "--all-features"])
                .expect("failed ark tests");
        }
        "modules" => {
            let client_path = format!("target/{}/ark-client", ctx.mode_str());
            ctx.call(&client_path, &["module", "test", "--sequential"])
                .expect("failed module tests");
        }
        unknown => panic!("unknown test target '{}'", unknown),
    }

    if which == "ark" && !ctx.skip_build {
        cargo::build_all(ctx);
        sccache::show(ctx, "build all");
    }
}

/// etc ...

The basic idea for this is pretty simple...

1. Provide "entry" scripts that compile a Rust executable and execute it if it isn't already available (a bad cargo run 😉)
2. No dependencies! since we run this for every single one of our CI build jobs, we want this to compile quickly, it currently takes ~1s to compile and start the binary atm
3. Rely on normal host tooling (curl, tar, cargo, etc, etc) for the actual operations, but avoid platform incompatibilities by using the usual stuff in the Rust std library like Command/std::fs/Path etc

Thanks!