Validate .claude/ follows "capabilities not workflows" — for calibrated LLM security research.

skills_validation.rs

//! Validation tests for the .claude/ directory structure.
//!
//! These tests enforce the layering model: GLOBAL CONTEXT is always-on and
//! non-procedural, AGENTS express perspective without workflows, SKILLS
//! describe capabilities without success criteria, and REFERENCES are
//! explicitly elective playbooks.
//!
//! ## What We Check
//!
//! **CLAUDE.md (Global Context)**
//! - No workflow verbs like "step 1", "first,", "then,", "finally,".
//! - No fenced code blocks that would embed procedures.
//!
//! **agents/*.md**
//! - Valid YAML frontmatter with name and description.
//! - Maximum 120 lines to keep agents lean.
//! - No fenced code blocks.
//! - No step-by-step procedural patterns.
//!
//! **skills/*/SKILL.md**
//! - Valid YAML frontmatter with name, description, and Capability section.
//! - Maximum 500 lines.
//! - Frontmatter contains only `name` and `description` keys.
//! - No success criteria terms like "must ensure" or "requirements:".
//! - No fenced code blocks or procedural markers.
//! - Skills with references/ directory have a References section.
//!
//! **skills/*/references/*.md**
//! - Must state "optional" near the top to be explicitly elective.

use std::fs;
use std::path::{Path, PathBuf};

/// Reads a file to string, panicking with a descriptive message on failure.
fn read_to_string(path: &Path) -> String {
    fs::read_to_string(path).unwrap_or_else(|e| {
        panic!("failed to read {}: {e}", path.display());
    })
}

/// Counts lines in a string, treating trailing newline correctly.
fn count_lines(s: &str) -> usize {
    s.lines().count()
}

/// Returns all subdirectories under the given root, sorted alphabetically.
fn all_skill_dirs(root: &Path) -> Vec<PathBuf> {
    let mut out = Vec::new();
    let entries = fs::read_dir(root).unwrap_or_else(|e| {
        panic!("failed to read_dir {}: {e}", root.display());
    });

    for entry in entries {
        let entry = entry.unwrap_or_else(|e| {
            panic!("failed to read_dir entry under {}: {e}", root.display());
        });
        let path = entry.path();
        if path.is_dir() {
            out.push(path);
        }
    }

    out.sort();
    out
}

/// Asserts that a SKILL.md file has valid YAML frontmatter with required keys.
fn assert_skill_md_has_frontmatter(skill_md: &str, path: &Path) {
    assert!(
        skill_md.starts_with("---\n"),
        "missing YAML frontmatter start in {}",
        path.display()
    );
    assert!(
        skill_md.contains("\nname:"),
        "missing `name:` in {} frontmatter",
        path.display()
    );
    assert!(
        skill_md.contains("\ndescription:"),
        "missing `description:` in {} frontmatter",
        path.display()
    );
    assert!(
        skill_md.contains("\n## Capability\n"),
        "missing `## Capability` section in {}",
        path.display()
    );
}

/// Asserts that frontmatter contains only allowed keys, rejecting directives.
///
/// This prevents embedding control-loop directives like `agent:` or `context:`
/// that would make SKILL/AGENT files procedural rather than declarative.
fn assert_frontmatter_keys_are_minimal(md: &str, path: &Path, allowed: &[&str]) {
    assert!(md.starts_with("---\n"), "missing frontmatter in {}", path.display());

    let mut keys = Vec::new();
    let mut seen_end = false;
    for (idx, line) in md.lines().enumerate() {
        if idx == 0 {
            continue;
        }
        if line == "---" {
            seen_end = true;
            break;
        }
        // Only consider top-level YAML keys (no indentation).
        if line.starts_with(' ') || line.starts_with('\t') {
            continue;
        }
        if let Some((key, _rest)) = line.split_once(':') {
            let key = key.trim();
            if !key.is_empty() {
                keys.push(key.to_string());
            }
        }
    }
    assert!(seen_end, "missing frontmatter end delimiter in {}", path.display());

    for key in keys {
        assert!(
            allowed.iter().any(|a| *a == key),
            "unexpected frontmatter key `{}` in {} (allowed: {:?})",
            key,
            path.display(),
            allowed
        );
    }
}

/// Checks if a line is under a markdown section containing the given substring.
fn line_is_in_section(lines: &[&str], idx: usize, section_substring: &str) -> bool {
    let needle = section_substring.to_lowercase();
    for j in (0..=idx).rev() {
        if let Some(rest) = lines[j].strip_prefix("## ") {
            return rest.to_lowercase().contains(&needle);
        }
    }
    false
}

/// Asserts that an agent markdown file has valid YAML frontmatter.
fn assert_agent_md_has_frontmatter(agent_md: &str, path: &Path) {
    assert!(
        agent_md.starts_with("---\n"),
        "missing YAML frontmatter start in {}",
        path.display()
    );
    assert!(
        agent_md.contains("\nname:"),
        "missing `name:` in {} frontmatter",
        path.display()
    );
    assert!(
        agent_md.contains("\ndescription:"),
        "missing `description:` in {} frontmatter",
        path.display()
    );
}

/// Verifies that skills directory exists with valid SKILL.md files.
#[test]
fn skills_structure_is_sane() {
    let skills_root = Path::new(".claude/skills");
    assert!(
        skills_root.is_dir(),
        "missing skills root at {}",
        skills_root.display()
    );

    let skill_dirs = all_skill_dirs(skills_root);
    assert!(
        skill_dirs.len() >= 2,
        "expected >=2 skill dirs under {}, got {}",
        skills_root.display(),
        skill_dirs.len()
    );

    for dir in skill_dirs {
        let skill_md_path = dir.join("SKILL.md");
        assert!(
            skill_md_path.is_file(),
            "missing SKILL.md in {}",
            dir.display()
        );

        let skill_md = read_to_string(&skill_md_path);
        assert_skill_md_has_frontmatter(&skill_md, &skill_md_path);

        let lines = count_lines(&skill_md);
        assert!(
            lines < 500,
            "SKILL.md too long ({} lines) in {}",
            lines,
            skill_md_path.display()
        );
    }
}

/// Verifies that agent files are lean and contain no code blocks.
#[test]
fn agents_are_minimal_and_declarative() {
    let agents_root = Path::new(".claude/agents");
    assert!(
        agents_root.is_dir(),
        "missing agents root at {}",
        agents_root.display()
    );

    let entries = fs::read_dir(agents_root).unwrap_or_else(|e| {
        panic!("failed to read_dir {}: {e}", agents_root.display());
    });

    let mut agent_files = Vec::new();
    for entry in entries {
        let entry = entry.unwrap_or_else(|e| {
            panic!(
                "failed to read_dir entry under {}: {e}",
                agents_root.display()
            );
        });
        let path = entry.path();
        if path.extension().and_then(|e| e.to_str()) == Some("md") {
            agent_files.push(path);
        }
    }
    agent_files.sort();
    assert!(
        !agent_files.is_empty(),
        "expected at least one agent markdown file under {}",
        agents_root.display()
    );

    for path in agent_files {
        let agent_md = read_to_string(&path);
        assert_agent_md_has_frontmatter(&agent_md, &path);

        // Agents are perspective/value layers. Keep them lean.
        let lines = count_lines(&agent_md);
        assert!(
            lines <= 120,
            "agent file too long ({} lines): {}",
            lines,
            path.display()
        );

        // Avoid embedded playbooks or command blocks.
        assert!(
            !agent_md.contains("```"),
            "agent file contains fenced code block: {}",
            path.display()
        );
    }
}

/// Ensures CLAUDE.md contains no step-by-step workflow language.
#[test]
fn global_context_has_no_workflow_verbs() {
    let claude_md = Path::new(".claude/CLAUDE.md");
    assert!(
        claude_md.is_file(),
        "missing CLAUDE.md at {}",
        claude_md.display()
    );

    let content = read_to_string(claude_md);

    // Workflow verbs that suggest step-by-step procedures.
    let workflow_verbs = [
        "must then",
        "next,",
        "step 1",
        "step 2",
        "first,",
        "second,",
        "finally,",
        "afterward",
        "subsequently",
    ];

    for verb in workflow_verbs {
        assert!(
            !content.to_lowercase().contains(verb),
            "CLAUDE.md contains workflow verb '{}' - global context should \
             only contain norms and facts, not procedures",
            verb
        );
    }
}

/// Ensures CLAUDE.md contains no fenced code blocks.
#[test]
fn global_context_has_no_fenced_code_blocks() {
    let claude_md = Path::new(".claude/CLAUDE.md");
    assert!(
        claude_md.is_file(),
        "missing CLAUDE.md at {}",
        claude_md.display()
    );

    let content = read_to_string(claude_md);
    assert!(
        !content.contains("```"),
        "CLAUDE.md contains fenced code block - global context should avoid \
         embedding executable procedures"
    );
}

/// Ensures SKILL.md files describe capabilities, not requirements.
#[test]
fn skill_files_have_no_success_criteria() {
    let skills_root = Path::new(".claude/skills");
    let skill_dirs = all_skill_dirs(skills_root);

    // Terms that suggest success criteria or execution requirements.
    let success_criteria_terms = [
        "success criteria",
        "must ensure",
        "must verify",
        "requirement:",
        "requirements:",
        "you must",
        "should ensure",
        "shall ensure",
    ];

    for dir in skill_dirs {
        let skill_md_path = dir.join("SKILL.md");
        let skill_md = read_to_string(&skill_md_path);

        for term in success_criteria_terms {
            assert!(
                !skill_md.to_lowercase().contains(term),
                "SKILL.md at {} contains success criteria term '{}' - \
                 skills should describe capabilities, not requirements",
                skill_md_path.display(),
                term
            );
        }
    }
}

/// Ensures skill frontmatter contains only name and description.
#[test]
fn skill_frontmatter_is_minimal_and_non_directive() {
    let skills_root = Path::new(".claude/skills");
    let skill_dirs = all_skill_dirs(skills_root);

    // Skills are optional affordances; keep frontmatter free of routing,
    // triggers, or model/tool constraints.
    let allowed_keys = ["name", "description"];

    for dir in skill_dirs {
        let skill_md_path = dir.join("SKILL.md");
        let skill_md = read_to_string(&skill_md_path);
        assert_frontmatter_keys_are_minimal(&skill_md, &skill_md_path, &allowed_keys);
    }
}

/// Ensures SKILL.md files have no code blocks or procedural markers.
#[test]
fn skill_files_have_no_embedded_playbooks() {
    let skills_root = Path::new(".claude/skills");
    let skill_dirs = all_skill_dirs(skills_root);

    // Procedural markers that belong in references, not in SKILL.md itself.
    let procedural_markers = [
        "## procedure",
        "## workflow",
        "## steps",
        "step 1",
        "step 2",
        "first,",
        "then,",
        "finally,",
        "checklist",
        "success criteria",
    ];

    for dir in skill_dirs {
        let skill_md_path = dir.join("SKILL.md");
        let skill_md = read_to_string(&skill_md_path);

        // Keep skills language-first; avoid embedding fenced command blocks.
        assert!(
            !skill_md.contains("```"),
            "SKILL.md at {} contains fenced code block - move procedures to \
             references/ and keep SKILL.md declarative",
            skill_md_path.display()
        );

        let lower = skill_md.to_lowercase();
        let lines: Vec<&str> = skill_md.lines().collect();
        for marker in procedural_markers {
            if !lower.contains(marker) {
                continue;
            }

            // Allow these terms only inside a `## References` section (rare,
            // but acceptable when describing a referenced protocol by name).
            for (i, line) in lines.iter().enumerate() {
                if line.to_lowercase().contains(marker) {
                    assert!(
                        line_is_in_section(&lines, i, "reference"),
                        "SKILL.md at {} contains procedural marker '{}' outside \
                         References section at line {}",
                        skill_md_path.display(),
                        marker,
                        i + 1
                    );
                }
            }
        }
    }
}

/// Ensures agent files express perspective, not step-by-step procedures.
#[test]
fn agent_files_have_no_step_by_step_procedures() {
    let agents_root = Path::new(".claude/agents");
    let entries = fs::read_dir(agents_root).unwrap_or_else(|e| {
        panic!("failed to read_dir {}: {e}", agents_root.display());
    });

    let mut agent_files = Vec::new();
    for entry in entries {
        let entry = entry.unwrap_or_else(|e| {
            panic!(
                "failed to read_dir entry under {}: {e}",
                agents_root.display()
            );
        });
        let path = entry.path();
        if path.extension().and_then(|e| e.to_str()) == Some("md") {
            agent_files.push(path);
        }
    }

    // Procedural patterns that suggest step-by-step workflows.
    let procedural_patterns = [
        "step 1:",
        "step 2:",
        "1.",
        "2.",
        "3.",
        "first,",
        "then,",
        "next,",
        "finally,",
        "## procedure",
        "## workflow",
        "## steps",
    ];

    for path in agent_files {
        let agent_md = read_to_string(&path);
        let content_lower = agent_md.to_lowercase();

        for pattern in procedural_patterns {
            // Allow numbered lists in references section only.
            if pattern.chars().next().unwrap().is_ascii_digit() || pattern.ends_with(',') {
                let lines: Vec<&str> = agent_md.lines().collect();
                for (i, line) in lines.iter().enumerate() {
                    let line_lower = line.to_lowercase();
                    if line_lower.contains(pattern) {
                        // Check if we're in a References section.
                        let mut in_references = false;
                        for j in (0..i).rev() {
                            if lines[j].starts_with("## ") {
                                if lines[j].to_lowercase().contains("reference") {
                                    in_references = true;
                                }
                                break;
                            }
                        }

                        assert!(
                            in_references,
                            "agent file {} contains procedural pattern '{}' \
                             outside References section at line {} - agents \
                             should express perspective and values, not \
                             step-by-step procedures",
                            path.display(),
                            pattern,
                            i + 1
                        );
                    }
                }
            } else if content_lower.contains(pattern) {
                panic!(
                    "agent file {} contains procedural pattern '{}' - agents \
                     should express perspective and values, not step-by-step \
                     procedures",
                    path.display(),
                    pattern
                );
            }
        }
    }
}

/// Collects all markdown files from references/ directories under skills.
fn reference_markdown_files_under(skills_root: &Path) -> Vec<PathBuf> {
    let mut out = Vec::new();
    for dir in all_skill_dirs(skills_root) {
        let refs_dir = dir.join("references");
        if !refs_dir.is_dir() {
            continue;
        }
        let entries = fs::read_dir(&refs_dir).unwrap_or_else(|e| {
            panic!("failed to read_dir {}: {e}", refs_dir.display());
        });
        for entry in entries {
            let entry = entry.unwrap_or_else(|e| {
                panic!(
                    "failed to read_dir entry under {}: {e}",
                    refs_dir.display()
                );
            });
            let path = entry.path();
            if path.extension().and_then(|e| e.to_str()) == Some("md") {
                out.push(path);
            }
        }
    }
    out.sort();
    out
}

/// Ensures reference files state they are optional near the top.
#[test]
fn reference_files_are_explicitly_elective() {
    let skills_root = Path::new(".claude/skills");
    let reference_files = reference_markdown_files_under(skills_root);
    assert!(
        !reference_files.is_empty(),
        "expected at least one reference markdown file under {}",
        skills_root.display()
    );

    for path in reference_files {
        let md = read_to_string(&path);
        let head = md
            .lines()
            .take(15)
            .collect::<Vec<_>>()
            .join("\n")
            .to_lowercase();
        assert!(
            head.contains("optional"),
            "reference file {} should explicitly say it's optional near the top",
            path.display()
        );
    }
}

/// Ensures skills with references/ directory have a References section.
#[test]
fn skills_with_references_dir_have_references_section() {
    let skills_root = Path::new(".claude/skills");
    for dir in all_skill_dirs(skills_root) {
        let refs_dir = dir.join("references");
        if !refs_dir.is_dir() {
            continue;
        }
        let skill_md_path = dir.join("SKILL.md");
        let skill_md = read_to_string(&skill_md_path);
        assert!(
            skill_md.contains("\n## References\n"),
            "skill {} has references/ dir but SKILL.md lacks a '## References' section",
            dir.display()
        );
    }
}