Headless Agentic CMS browser smoke testing for Silverstripe

cms-browser-check.js

#!/usr/bin/env node

/**
 * Headless CMS browser smoke test for local Silverstripe admin pages. Useful when performing testing of the
 * Silverstripe CMS via coding agents such as Claude or Codex. Includes full rendering and JavaScript rendering by
 * running headless Chrome.
 *
 * IMPORTANT:
 * - This was vibe-coded (GPT-5.5 xhigh).
 * - Originally built for SS4 but may work with later versions if modified.
 *
 * INSTRUCTIONS:
 *
 * - Ensure you have Chrome/Chromium already installed on the machine that this script will be running on.
 * - Copy this file to your repository to `scripts/cms-browser-check` (without the .js extension, only here for gists)
 * - Be sure to update the "YOUR_*_HERE" stuff below!
 * - Adjust as needed.
 *
 * EXAMPLE ENTRY FOR AGENTS.md FILE:
 *
 * - For local CMS browser smoke tests with JavaScript execution, use `cms-browser-check <url> [url...]` from the
 *   host. It logs into the CMS with the standard local admin credentials (`YOUR_CMS_USERNAME_HERE` / `YOUR_CMS_PASSWORD_HERE`), launches headless Chrome via
 *   DevTools, captures runtime exceptions and console errors, and reports useful page state. The login step intentionally
 *   uses the same user agent as Chrome because the custom Silverstripe session implementation fingerprints sessions by
 *   user agent. In sandboxed agent runs, this may require escalated local-network access to reach the Docker/Traefik HTTP
 *   route.
 *
 * @author  Patrick Nelson (pat@catchyour.com)
 * @license MIT
 * @since   2026-07-20
 */

import { spawn, spawnSync } from 'node:child_process';
import { mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

const defaultHost = 'YOUR_CMS_HOST_HERE';
const defaultHostIp = '127.0.0.1';
const defaultUsername = 'YOUR_CMS_USERNAME_HERE';
const defaultPassword = 'YOUR_CMS_PASSWORD_HERE';
const defaultSessionCookieNames = ['PHPSESSID'];
const defaultSettleMs = 6000;
const defaultChromeUserAgent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.184 Safari/537.36';

function usage() {
	console.log(`Usage:
  scripts/cms-browser-check [options] <cms-url> [cms-url...]

Options:
  --host <host>          CMS host name. Default: ${defaultHost}
  --host-ip <ip>         IP Chrome and login requests should use for the host. Default: ${defaultHostIp}
  --username <user>      CMS username. Default: ${defaultUsername}
  --password <pass>      CMS password. Default: ${defaultPassword}
  --session-cookie <name>
                         Cookie name that indicates login succeeded. Can be used more than once.
                         Default: ${defaultSessionCookieNames.join(', ')}
  --fail-selector <css>  CSS selector that should fail the check if it matches anything.
                         Can be used more than once.
  --chrome <path>        Chrome/Chromium binary. Auto-detected by default.
  --user-agent <ua>      User agent used for both login and headless Chrome.
  --settle-ms <ms>       Milliseconds to wait after each navigation. Default: ${defaultSettleMs}
  --include-cookie-warnings
                         Include Chrome third-party-cookie warnings in reported events.
  --help                 Show this help.

Examples:
  scripts/cms-browser-check http://YOUR_CMS_HOST_HERE/admin/pages/edit/show/YOUR_PAGE_ID_HERE
  scripts/cms-browser-check /admin/pages/edit/show/YOUR_PAGE_ID_HERE /admin/pages/edit/EditForm/YOUR_PAGE_ID_HERE/field/YOUR_RELATION_FIELD_HERE/item/YOUR_ITEM_ID_HERE/edit
  scripts/cms-browser-check --session-cookie YOUR_SESSION_COOKIE_NAME_HERE --fail-selector '[data-config="YOUR_CONFIG_KEY_HERE"]' /admin/pages/edit/show/YOUR_PAGE_ID_HERE
`);
}

function parseArgs(argv) {
	const options = {
		host: defaultHost,
		hostIp: defaultHostIp,
		username: defaultUsername,
		password: defaultPassword,
		sessionCookieNames: [...defaultSessionCookieNames],
		failSelectors: [],
		chrome: process.env.CMS_BROWSER_CHROME || '',
		userAgent: process.env.CMS_BROWSER_USER_AGENT || defaultChromeUserAgent,
		settleMs: defaultSettleMs,
		includeCookieWarnings: false,
		urls: [],
	};

	for (let i = 0; i < argv.length; i += 1) {
		const arg = argv[i];
		if (arg === '--help' || arg === '-h') {
			usage();
			process.exit(0);
		}
		if (arg === '--include-cookie-warnings') {
			options.includeCookieWarnings = true;
			continue;
		}
		if (arg.startsWith('--')) {
			const value = argv[i + 1];
			if (!value || value.startsWith('--')) {
				throw new Error(`Missing value for ${arg}`);
			}
			i += 1;
			if (arg === '--host') options.host = value;
			else if (arg === '--host-ip') options.hostIp = value;
			else if (arg === '--username') options.username = value;
			else if (arg === '--password') options.password = value;
			else if (arg === '--session-cookie') options.sessionCookieNames.push(value);
			else if (arg === '--fail-selector') options.failSelectors.push(value);
			else if (arg === '--chrome') options.chrome = value;
			else if (arg === '--user-agent') options.userAgent = value;
			else if (arg === '--settle-ms') options.settleMs = Number.parseInt(value, 10);
			else throw new Error(`Unknown option: ${arg}`);
			continue;
		}
		options.urls.push(arg);
	}

	if (!Number.isFinite(options.settleMs) || options.settleMs < 0) {
		throw new Error('--settle-ms must be a non-negative integer');
	}
	if (!options.urls.length) {
		usage();
		throw new Error('At least one CMS URL is required');
	}
	options.sessionCookieNames = options.sessionCookieNames.filter((name) => (
		name && !name.startsWith('YOUR_')
	));

	options.urls = options.urls.map((url) => normalizeUrl(url, options.host));
	return options;
}

function normalizeUrl(url, host) {
	if (url.startsWith('http://') || url.startsWith('https://')) return url;
	if (!url.startsWith('/')) return `http://${host}/${url}`;
	return `http://${host}${url}`;
}

function firstPath(url) {
	const parsed = new URL(url);
	return `${parsed.pathname}${parsed.search}${parsed.hash}`;
}

function findChromeBinary(explicitPath) {
	if (explicitPath) return explicitPath;

	const candidates = [
		'google-chrome',
		'google-chrome-stable',
		'chromium',
		'chromium-browser',
	];

	for (const candidate of candidates) {
		const result = spawnSync('which', [candidate], { encoding: 'utf8' });
		if (result.status === 0 && result.stdout.trim()) return result.stdout.trim();
	}

	throw new Error('Could not find Chrome/Chromium. Install Chrome or pass --chrome /path/to/browser.');
}

function sleep(ms) {
	return new Promise((resolve) => setTimeout(resolve, ms));
}

function cookiePairs(cookieJar) {
	return [...cookieJar.entries()].map(([name, cookie]) => `${name}=${cookie.value}`).join('; ');
}

function parseSetCookieHeader(header) {
	const firstSemicolon = header.indexOf(';');
	const pair = firstSemicolon === -1 ? header : header.slice(0, firstSemicolon);
	const equalsIndex = pair.indexOf('=');
	if (equalsIndex === -1) return null;
	return {
		name: pair.slice(0, equalsIndex).trim(),
		value: pair.slice(equalsIndex + 1).trim(),
		httpOnly: /;\s*httponly\b/i.test(header),
		secure: /;\s*secure\b/i.test(header),
		path: /;\s*path=([^;]+)/i.exec(header)?.[1] || '/',
	};
}

function splitSetCookieHeader(header) {
	if (!header) return [];

	const parts = [];
	let start = 0;
	let inExpires = false;
	for (let i = 0; i < header.length; i += 1) {
		const chunk = header.slice(Math.max(0, i - 8), i + 1).toLowerCase();
		if (chunk.endsWith('expires=')) inExpires = true;
		if (inExpires && header[i] === ';') inExpires = false;
		if (!inExpires && header[i] === ',' && /\s*[^=;,]+=/.test(header.slice(i + 1, i + 80))) {
			parts.push(header.slice(start, i).trim());
			start = i + 1;
		}
	}
	parts.push(header.slice(start).trim());
	return parts.filter(Boolean);
}

function setCookiesFromResponse(response, cookieJar) {
	const headers = typeof response.headers.getSetCookie === 'function'
		? response.headers.getSetCookie()
		: splitSetCookieHeader(response.headers.get('set-cookie'));

	for (const header of headers) {
		const cookie = parseSetCookieHeader(header);
		if (!cookie?.name) continue;
		if (cookie.value === 'deleted') {
			cookieJar.delete(cookie.name);
			continue;
		}
		cookieJar.set(cookie.name, cookie);
	}
}

async function fetchLocal(path, options, requestOptions = {}) {
	const url = `http://${options.host}${path}`;
	const headers = {
		'User-Agent': options.userAgent,
		...(requestOptions.headers || {}),
	};

	const response = await fetch(url, {
		redirect: 'manual',
		...requestOptions,
		headers,
	});

	if (options.cookieJar) setCookiesFromResponse(response, options.cookieJar);
	return response;
}

function extractInputValue(html, name, fallback = '') {
	const pattern = new RegExp(`<input[^>]+name=["']${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["'][^>]*>`, 'i');
	const tag = pattern.exec(html)?.[0] || '';
	return /value=["']([^"']*)["']/i.exec(tag)?.[1] || fallback;
}

async function login(options) {
	const cookieJar = new Map();
	const loginOptions = { ...options, cookieJar };
	const backUrl = firstPath(options.urls[0]);
	const loginPath = `/Security/login?BackURL=${encodeURIComponent(backUrl)}`;
	const loginResponse = await fetchLocal(loginPath, loginOptions);
	const loginHtml = await loginResponse.text();
	const securityId = extractInputValue(loginHtml, 'SecurityID');
	const authMethod = extractInputValue(
		loginHtml,
		'AuthenticationMethod',
		'SilverStripe\\Security\\MemberAuthenticator\\MemberAuthenticator'
	);

	if (!securityId) {
		throw new Error('Could not find SecurityID on login page');
	}

	const body = new URLSearchParams();
	body.set('AuthenticationMethod', authMethod);
	body.set('Email', options.username);
	body.set('Password', options.password);
	body.set('BackURL', backUrl);
	body.set('SecurityID', securityId);
	body.set('action_doLogin', 'Log in');

	const postResponse = await fetchLocal('/Security/login/default/LoginForm/', loginOptions, {
		method: 'POST',
		headers: {
			'Content-Type': 'application/x-www-form-urlencoded',
			Cookie: cookiePairs(cookieJar),
		},
		body,
	});

	if (![200, 302, 303].includes(postResponse.status)) {
		throw new Error(`Login POST failed: ${postResponse.status} ${postResponse.statusText}`);
	}

	if (options.sessionCookieNames.length && !options.sessionCookieNames.some((name) => cookieJar.has(name))) {
		throw new Error('Login did not produce expected PHP session cookies');
	}

	return cookieJar;
}

async function fetchJson(url, options = {}) {
	const response = await fetch(url, options);
	if (!response.ok) {
		throw new Error(`${options.method || 'GET'} ${url} failed: ${response.status} ${response.statusText}`);
	}
	return response.json();
}

async function waitForChrome(port) {
	const deadline = Date.now() + 15000;
	let lastError;
	while (Date.now() < deadline) {
		try {
			return await fetchJson(`http://127.0.0.1:${port}/json/version`);
		} catch (error) {
			lastError = error;
			await sleep(250);
		}
	}
	throw lastError || new Error('Chrome did not expose DevTools');
}

function connect(webSocketDebuggerUrl) {
	if (typeof WebSocket === 'undefined') {
		throw new Error('Global WebSocket is not available. Use Node 22+ or a Node build with WebSocket support.');
	}

	const ws = new WebSocket(webSocketDebuggerUrl);
	let nextId = 1;
	const pending = new Map();
	const events = [];

	ws.addEventListener('message', (message) => {
		const data = JSON.parse(message.data);
		if (data.id && pending.has(data.id)) {
			const { resolve, reject } = pending.get(data.id);
			pending.delete(data.id);
			if (data.error) reject(new Error(`${data.error.message}: ${JSON.stringify(data.error.data || '')}`));
			else resolve(data.result || {});
			return;
		}
		if (data.method) events.push(data);
	});

	const opened = new Promise((resolve, reject) => {
		ws.addEventListener('open', resolve, { once: true });
		ws.addEventListener('error', reject, { once: true });
	});

	function send(method, params = {}) {
		const id = nextId;
		nextId += 1;
		const promise = new Promise((resolve, reject) => pending.set(id, { resolve, reject }));
		ws.send(JSON.stringify({ id, method, params }));
		return promise;
	}

	return { ws, opened, send, events };
}

async function evaluate(cdp, expression) {
	const result = await cdp.send('Runtime.evaluate', {
		expression,
		awaitPromise: true,
		returnByValue: true,
	});
	if (result.exceptionDetails) {
		throw new Error(result.exceptionDetails.text || 'Runtime.evaluate failed');
	}
	return result.result?.value;
}

async function maybeLoginInBrowser(cdp, options) {
	const didSubmit = await evaluate(cdp, `(() => {
		if (!/\\/Security\\/login/.test(location.pathname)) return false;
		const email = document.querySelector('input[name="Email"], input[type="email"]');
		const password = document.querySelector('input[name="Password"], input[type="password"]');
		if (!email || !password) return false;
		email.value = ${JSON.stringify(options.username)};
		email.dispatchEvent(new Event('input', { bubbles: true }));
		email.dispatchEvent(new Event('change', { bubbles: true }));
		password.value = ${JSON.stringify(options.password)};
		password.dispatchEvent(new Event('input', { bubbles: true }));
		password.dispatchEvent(new Event('change', { bubbles: true }));
		const form = password.form || email.form;
		const submit = form?.querySelector('button[type="submit"], input[type="submit"]');
		if (submit) submit.click();
		else form?.submit();
		return true;
	})()`);
	if (didSubmit) await sleep(options.settleMs);
	return didSubmit;
}

function summarizeEvents(events, options) {
	const interesting = [];
	for (const event of events) {
		if (event.method === 'Runtime.exceptionThrown') {
			const details = event.params?.exceptionDetails;
			interesting.push({
				type: 'exception',
				text: details?.text,
				description: details?.exception?.description,
				url: details?.url,
				line: details?.lineNumber,
				column: details?.columnNumber,
			});
		}
		if (event.method === 'Log.entryAdded') {
			const entry = event.params?.entry;
			if (!options.includeCookieWarnings && entry?.text?.includes('Third-party cookie will be blocked')) continue;
			if (entry?.text?.includes('was preloaded using link preload but not used')) continue;
			if (['error', 'warning'].includes(entry?.level)) {
				interesting.push({
					type: `log:${entry.level}`,
					text: entry.text,
					url: entry.url,
					line: entry.lineNumber,
				});
			}
		}
		if (event.method === 'Runtime.consoleAPICalled') {
			if (['error', 'warning', 'assert'].includes(event.params?.type)) {
				interesting.push({
					type: `console:${event.params.type}`,
					text: (event.params.args || []).map((arg) => arg.value || arg.description || '').join(' '),
				});
			}
		}
		if (event.method === 'Network.loadingFailed') {
			if (event.params?.canceled || event.params?.errorText === 'net::ERR_ABORTED') continue;
			interesting.push({
				type: 'network:failed',
				text: event.params?.errorText,
				url: event.params?.requestId,
			});
		}
	}
	return interesting;
}

function isFailureEvent(event) {
	return [
		'exception',
		'console:error',
		'console:assert',
		'log:error',
		'network:failed',
	].includes(event.type);
}

function chromeHostResolverRules(options) {
	const hosts = new Set([options.host]);
	if (!options.host.startsWith('static.')) hosts.add(`static.${options.host}`);
	return [...hosts].map((host) => `MAP ${host} ${options.hostIp}`).join(', ');
}

async function runBrowser(options, cookieJar) {
	const chrome = findChromeBinary(options.chrome);
	const port = 9222 + Math.floor(Math.random() * 1000);
	const userDataDir = await mkdtemp(join(tmpdir(), 'silverstripe-cms-browser-'));
	const chromeStderr = [];

	const browser = spawn(chrome, [
		'--headless=new',
		'--disable-gpu',
		'--disable-dev-shm-usage',
		'--no-sandbox',
		'--remote-debugging-address=127.0.0.1',
		`--remote-debugging-port=${port}`,
		`--user-data-dir=${userDataDir}`,
		`--user-agent=${options.userAgent}`,
		`--host-resolver-rules=${chromeHostResolverRules(options)}`,
		'about:blank',
	], {
		stdio: ['ignore', 'ignore', 'pipe'],
	});

	browser.stderr.on('data', (chunk) => chromeStderr.push(String(chunk)));

	try {
		await waitForChrome(port);
		const target = await fetchJson(`http://127.0.0.1:${port}/json/new?about:blank`, { method: 'PUT' });
		const cdp = connect(target.webSocketDebuggerUrl);
		await cdp.opened;
		await cdp.send('Page.enable');
		await cdp.send('Runtime.enable');
		await cdp.send('Log.enable');
		await cdp.send('Network.enable');

		const setCookieResults = [];
		for (const cookie of cookieJar.values()) {
			setCookieResults.push(await cdp.send('Network.setCookie', {
				name: cookie.name,
				value: cookie.value,
				path: cookie.path || '/',
				httpOnly: cookie.httpOnly,
				secure: cookie.secure,
				url: `http://${options.host}/`,
			}));
		}

		const results = [];
		for (const url of options.urls) {
			cdp.events.length = 0;
			await cdp.send('Page.navigate', { url });
			await sleep(options.settleMs);
			const didSubmitLogin = await maybeLoginInBrowser(cdp, options);
			if (/\/Security\/login/.test(await evaluate(cdp, 'location.pathname'))) {
				await cdp.send('Page.navigate', { url });
				await sleep(options.settleMs);
			}

			const pageState = await evaluate(cdp, `(() => ({
				url: location.href,
				title: document.title,
				readyState: document.readyState,
				bodyTextStart: (document.body?.innerText || '').slice(0, 500),
				loadingTextPresent: /\\bLoading\\b/.test(document.body?.innerText || ''),
				failSelectorMatches: ${JSON.stringify(options.failSelectors)}.map((selector) => {
					try {
						return { selector, count: document.querySelectorAll(selector).length };
					} catch (error) {
						return { selector, count: 0, error: error.message };
					}
				}),
				tinyMCEFieldCount: document.querySelectorAll('[data-editor="tinyMCE"]').length,
				loginMessage: document.querySelector('.message, .alert, .bad, .good, .notice')?.innerText || ''
			}))()`);

			results.push({
				inputUrl: url,
				didSubmitLogin,
				pageState,
				events: summarizeEvents(cdp.events, options),
			});
		}

		cdp.ws.close();
		return {
			chrome,
			setCookieResults,
			results,
			chromeStderr: chromeStderr.slice(-5),
		};
	} finally {
		browser.kill('SIGTERM');
	}
}

async function main() {
	const options = parseArgs(process.argv.slice(2));
	const cookieJar = await login(options);
	const report = await runBrowser(options, cookieJar);
	const hasFailureEvents = report.results.some((result) => result.events.some(isFailureEvent));
	const hasBadState = report.results.some((result) => (
		result.pageState.loadingTextPresent
		|| result.pageState.failSelectorMatches.some((match) => match.error || match.count > 0)
		|| /\/Security\/login/.test(new URL(result.pageState.url).pathname)
	));

	console.log(JSON.stringify(report, null, 2));
	process.exit(hasFailureEvents || hasBadState ? 1 : 0);
}

main().catch((error) => {
	console.error(error.stack || error.message);
	process.exit(1);
});