_Vet.md

This script is a vetting tool designed to automate the process of running tests, retrying failures if necessary, and saving the results of each test cycle to a session-specific directory. Here's a summary of its functionality:

Key Features and Workflow:

1. Session Setup:

   • Creates a session-specific directory based on the current timestamp and Git branch name.
   • Ensures the base directory (vetting-results) exists and creates the session directory recursively.

2. User-Specified Test Parsing:

   • Parses command-line arguments to extract user-specified tests to run.

3. Initial Test Execution:

   • Runs a specified set of tests (or all tests if not specified).
   • Saves results of the test run.

4. Retry Logic:

   • If some tests fail, the script retries the failed tests within a specified timeout period (2 hours).
   • Results are periodically saved during the retry process.

5. Result Saving:

   • Saves the test results into the session-specific directory for future review.
   • Test results are saved in sub-folders (e.g., initial and retry cycles).

6. Branch Name Determination:

   • Uses Git to determine the name of the current branch.
   • Defaults to "unknown-branch" if the branch name cannot be retrieved.

7. Error Handling and Exit Codes:

   • Provides user feedback and exit codes depending on test outcomes:
     • Successful run: exits with code 0.
     • Errors during test execution or retries: exits with code 1.

8. Modular Functions:

   • setupSessionDirectory: Handles session directory setup.
   • saveTestResults: Responsible for saving test results.
   • parseUserSpecifiedTests: Extracts user-provided test specifiers from command-line arguments.
   • executeInitialTestRun: Runs the tests initially and stores results.
   • getBranchName: Fetches the current Git branch name while handling errors gracefully.

Tools and Dependencies:

• Bun: Used to spawn child processes (e.g., for Git commands).
• fs and fs/promises: For file system operations like directory creation and copying results.
• path: For constructing file and directory paths.

Error Management:

• Provides detailed error reporting at various stages of script execution, such as during directory creation, branch name retrieval, or saving results.

This script ensures robust handling of automated test runs, accommodating retries and maintaining organized results for review. It is particularly useful for continuous integration pipelines or environments where test reliability is critical.

vet.ts

#!/usr/bin/env bun

import { existsSync } from 'fs';
import { join } from 'path';
import { spawn } from 'bun';
import { cp, mkdir } from 'fs/promises';

main().catch((error) => {
  console.error(`Script error: ${error}`);
  process.exit(1);
});

async function main() {
  const retryTimeout = 2 * 60 * 60 * 1000; // 2 hours in milliseconds
  const label = 'Vetting script completed';
  console.time(label);

  const sessionDir = await setupSessionDirectory();

  const userSpecifiedTests = parseUserSpecifiedTests();
  const initialResult = await executeInitialTestRun(
    userSpecifiedTests,
    sessionDir,
  );

  if (initialResult.success) {
    console.log('All tests passed! 🎉');
    console.timeEnd(label);
    process.exit(0);
  }

  if (!initialResult.lastRun) {
    console.error('Failed to read initial test results');
    console.timeEnd(label);
    process.exit(1);
  }

  const allTestsPassed = await handleRetryProcess(
    initialResult.lastRun,
    sessionDir,
    retryTimeout,
  );
  console.timeEnd(label);
  if (allTestsPassed) {
    process.exit(0);
  } else {
    process.exit(1);
  }
}

async function setupSessionDirectory(): Promise<string> {
  const vettingResultsPath = await generateSessionDirectoryPath();

  try {
    await setupVettingResultsDirectory();
    await mkdir(vettingResultsPath, { recursive: true });
    console.log(`Created session directory: ${vettingResultsPath}`);
  } catch (error) {
    console.error(`Failed to create session directory: ${error.message}`);
    process.exit(1);
  }

  return vettingResultsPath;

  async function generateSessionDirectoryPath() {
    const branchName = await getBranchName();
    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
    const sessionDirName = `${timestamp}-${branchName}`;
    return join(process.cwd(), 'vetting-results', sessionDirName);
  }

  async function setupVettingResultsDirectory() {
    const basePath = join(process.cwd(), 'vetting-results');
    if (!existsSync(basePath)) {
      await mkdir(basePath, { recursive: true });
    }
  }
}

async function saveTestResults(
  sessionDir: string,
  cycleName: string,
): Promise<void> {
  console.log(`Saving ${cycleName} test results...`);
  const testResultsPath = join(process.cwd(), 'test-results');
  if (!existsSync(testResultsPath)) {
    console.warn('Test results directory does not exist. Skipping save.');
    return;
  }
  const destPath = join(sessionDir, cycleName);

  try {
    if (!existsSync(destPath)) {
      await mkdir(destPath);
    }

    await cp(testResultsPath, destPath, { recursive: true });

    console.log(`Test results saved to: ${destPath}`);
  } catch (error) {
    console.error(`Failed to save test results: ${error.message}`);
  }
}

async function getBranchName(): Promise<string> {
  try {
    const proc = Bun.spawn(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], {
      stdout: 'pipe',
      stderr: 'pipe',
    });

    const output = await new Response(proc.stdout).text();
    await proc.exited;

    if (proc.exitCode !== 0 || !output.trim()) {
      throw new Error('Failed to retrieve branch name');
    }

    return output
      .trim()
      .replace(/[^a-zA-Z0-9-_]/g, '-') // Replace invalid characters
      .toLowerCase();
  } catch (error) {
    console.warn(
      'Failed to identify branch name. Defaulting to "unknown-branch".',
      error,
    );
    return 'unknown-branch';
  }
}

function parseUserSpecifiedTests(): string[] {
  const args = process.argv.slice(2); // Skip the first two arguments (node and script path)
  const testSpecifiers = args.filter((arg) => !arg.startsWith('--'));
  if (testSpecifiers.length > 0) {
    console.log(`User-specified tests: ${testSpecifiers.join(', ')}`);
  }
  return testSpecifiers;
}

async function executeInitialTestRun(
  specifiedTests: string[],
  sessionDir: string,
): Promise<TestResult> {
  displayTestRunMessage();
  const success = await runTests(false, specifiedTests);
  await saveTestResults(sessionDir, 'initial-run');

  const lastRun = await getLastRunInfo();

  return { success, lastRun };

  function displayTestRunMessage(testCount: number = specifiedTests.length) {
    console.log(
      testCount > 0
        ? 'Running specified tests...'
        : 'Running full test suite...',
    );
  }
}

async function handleRetryProcess(
  previousRun: LastRun,
  sessionDir: string,
  retryTimeout: number,
): Promise<boolean> {
  console.log(
    'Some tests failed. Starting retry process with single worker...',
  );

  let retryCycle = 1;
  const startTime = Date.now();
  let allTestsPassed = false;

  while (Date.now() - startTime < retryTimeout) {
    const retryResult = await executeRetry();
    await saveTestResults(sessionDir, `retry-cycle-${retryCycle}`);

    if (retryResult.success) {
      console.log('All retried tests passed! 🎉');
      allTestsPassed = true;
      break;
    }

    if (!retryResult.lastRun) {
      throw new Error('Failed to read the latest test results');
    }

    if (await areFailuresConsistent(retryResult.lastRun, previousRun)) {
      console.log('Same tests failed twice in a row. Stopping retry process.');
      console.log('Failed tests:', retryResult.lastRun.failedTests.join(', '));
      allTestsPassed = false;
      break;
    }

    previousRun = retryResult.lastRun;
    retryCycle++;
    console.log('Different test failures detected, retrying...');
  }
  return allTestsPassed;
}

async function executeRetry(): Promise<TestResult> {
  const success = await runTests(true);
  const lastRun = await getLastRunInfo();

  return { success, lastRun };
}

async function areFailuresConsistent(
  currentRun: LastRun,
  previousRun: LastRun,
): Promise<boolean> {
  if (currentRun.failedTests.length !== previousRun.failedTests.length)
    return false;

  const sortedCurrent = [...currentRun.failedTests].sort();
  const sortedPrevious = [...previousRun.failedTests].sort();

  return sortedCurrent.every((test, index) => test === sortedPrevious[index]);
}

async function runTests(
  lastFailedOnly: boolean = false,
  specifiedTests: string[] = [],
): Promise<boolean> {
  const args = buildTestArgs(lastFailedOnly, specifiedTests);
  const proc = spawnTestProcess(args);

  const exitCode = await proc.exited;
  return exitCode === 0;
}

function buildTestArgs(
  lastFailedOnly: boolean,
  specifiedTests: string[],
): string[] {
  const args = ['playwright', 'test'];

  if (lastFailedOnly) {
    // Use `--last-failed` for retries
    args.push('--last-failed', '--workers', '1', '--retries', '1');
  } else {
    // Use specified test filters only for the initial run
    args.push('--workers', '4', '--retries', '1');
    if (specifiedTests.length > 0) {
      args.push(...specifiedTests); // Add specific tests for initial run
    }
  }

  return args;
}

function spawnTestProcess(args: string[]) {
  console.log(`Running tests with command: bunx ${args.join(' ')}`);

  return spawn(['bunx', ...args], {
    stdout: 'inherit',
    stderr: 'inherit',
  });
}

async function getLastRunInfo(): Promise<LastRun | null> {
  try {
    const file = Bun.file(
      join(process.cwd(), 'test-results', '.last-run.json'),
    );
    const exists = await file.exists();

    if (!exists) {
      return null;
    }

    const content = await file.text();
    return JSON.parse(content) as LastRun;
  } catch {
    return null;
  }
}

interface LastRun {
  status: string;
  failedTests: string[];
}

interface TestResult {
  success: boolean;
  lastRun: LastRun | null;
}