PROTOTYPE PortSwigger Burp extension which loads a Yaml file containing regular expressions for sensitive information in order to automatically identify and flag

BurpExtender.java

//
//
// PROTOTYPE Passive Sensitive Information Observer
//
// test with https://ginandjuice.shop/
//
// This has been tested with Burp Suite Pro to trial the concept.
//
// Ollie Whitehouse - @ollieatnowhere
//

package burp;

import burp.BurpExtender.MyExtensionUnloadHandler;
import burp.api.montoya.BurpExtension;
import burp.api.montoya.MontoyaApi;
import burp.api.montoya.logging.Logging;
import burp.api.montoya.extension.Extension;
import burp.api.montoya.extension.ExtensionUnloadingHandler;

public class BurpExtender implements BurpExtension
{
	private Logging logging;
	  
	@Override
	public void initialize(MontoyaApi api)
	{
	  
		Extension extension = api.extension();
	  
		// set extension name
		api.extension().setName("Passive Sensitive Data Detector");

		logging = api.logging();

		// register a new extension unload handler
		extension.registerUnloadingHandler(new MyExtensionUnloadHandler());
      
		// write a message to our output stream
		logging.logToOutput("Passive Sensitive Data Detector");
			
		// register the scan check
		api.scanner().registerScanCheck(new MyScanCheck(api,logging));
	}
  
	// Handler
	public class MyExtensionUnloadHandler implements ExtensionUnloadingHandler {
		@Override
		public void extensionUnloaded() {
			logging.logToOutput("Extension was unloaded.");
		}
	}

DefaultPassiveRules.txt

patterns:
  - pattern:
      name: Ollie Test Rule 1
      regex: ^.*Gin.*$
      confidence: high
  - pattern:
      name: Ollie Test Rule 2
      regex: ^.*Shop.*$
      confidence: high
  

MyScanCheck.java

//
// PROTOTYPE Passive Sensitive Information Observer
//
// test with https://ginandjuice.shop/
//
// This has been tested with Burp Suite Pro to trial the concept.
//
// Ollie Whitehouse - @ollieatnowhere
//
//

package burp;

import java.net.URI;
import java.util.*;
import java.util.regex.*;  
import java.net.http.*;
import java.net.http.HttpRequest.BodyPublisher;
import java.net.http.HttpResponse.BodyHandlers;
import java.time.Duration;
import java.time.LocalDateTime;
import static java.util.Collections.emptyList;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;

import org.yaml.snakeyaml.*;
import org.yaml.snakeyaml.constructor.Constructor;

import burp.api.montoya.MontoyaApi;
import burp.api.montoya.http.message.HttpRequestResponse;
import burp.api.montoya.http.message.requests.HttpRequest;
import burp.api.montoya.scanner.AuditResult;
import static burp.api.montoya.scanner.AuditResult.auditResult;
import static burp.api.montoya.scanner.audit.issues.AuditIssue.auditIssue;
import static burp.api.montoya.scanner.ConsolidationAction.KEEP_BOTH;
import static burp.api.montoya.scanner.ConsolidationAction.KEEP_EXISTING;
import burp.api.montoya.scanner.ConsolidationAction;
import burp.api.montoya.scanner.ScanCheck;
import burp.api.montoya.logging.Logging;
import burp.api.montoya.scanner.audit.insertionpoint.AuditInsertionPoint;
import burp.api.montoya.scanner.audit.issues.AuditIssue;
import burp.api.montoya.scanner.audit.issues.AuditIssueConfidence;
import burp.api.montoya.scanner.audit.issues.AuditIssueSeverity;
import burp.api.montoya.sitemap.SiteMap;

//
// The scan check
//
class MyScanCheck implements ScanCheck
{
    private final MontoyaApi api;
    private final Logging logging;
    private boolean rulesloaded = false;
    private List<Rule> rules = new ArrayList<Rule>();
   
    public static void readYml(Map<String,Object> map, Logging logging,List<Rule> Rules ){
        
    	if(Objects.isNull(map) || map.entrySet().size()==0){
            return;
        }
        
        for(Map.Entry<String,Object> entry: map.entrySet()){
            try {
                if (entry.getValue() instanceof List) {
                    for(int i = 0 ; i< ((List<?>) entry.getValue()).size();i++) {
                        Map<String, Object> casted = (Map<String, Object>) ((List<?>) entry.getValue()).get(i);
                        readYml(casted,logging, Rules);
                    }
                }
                else {
                	
                	//logging.logToOutput(entry.getKey() +" - "+entry.getValue());
                	//logging.logToOutput("" + entry.getValue() );
                	Map<String, Object> pattern = (Map<String, Object>)entry.getValue();
                	
                	//logging.logToOutput("" + pattern.get("name"));
                	//logging.logToOutput("" + pattern.get("regex"));
                	//logging.logToOutput("" + pattern.get("confidence"));

                	Rule myNewRule = new Rule();
                	
                	myNewRule.name = pattern.get("name").toString();
                	myNewRule.regex = pattern.get("regex").toString();
                	myNewRule.confidence = pattern.get("confidence").toString();
                	Rules.add(myNewRule);
                	
                }
            }catch (Exception ex){
            	logging.logToError(ex.getMessage());
            }
        }

    }
    
    //
    // Loads the rules from disk
    //
    private boolean LoadRules() {
    
    	// Filename
		String PASSIVERULESFILE = System.getenv("PASSIVERULEFILE");
		if(PASSIVERULESFILE== null || PASSIVERULESFILE.isEmpty()) {
			PASSIVERULESFILE = ".\\DefaultPassiveRules.txt";
			
			logging.logToError("No passive rules file supplied via environment variable \'PASSIVERULEFILE\' using - " + PASSIVERULESFILE + " in " + System.getProperty("user.dir"));
			logging.logToOutput("No passive rules file supplied via environment variable \'PASSIVERULEFILE\' using - " + PASSIVERULESFILE + " in " + System.getProperty("user.dir"));
		} else {		
			logging.logToOutput("Passive rules file - " + PASSIVERULESFILE); 
		}
		
		// Check it exists - if not download
		File f = new File(PASSIVERULESFILE);
		if(f.exists() == false) {
			logging.logToError("Passive rules file doesn't exist - " + PASSIVERULESFILE);
			return false;
		}
	
		Yaml yaml = new Yaml();
		
		// Load it
		try {
	    	InputStream inputStream = new FileInputStream(new File(PASSIVERULESFILE));
	    	    	
	        Map<String, Object> data = yaml.load(inputStream);
	        readYml(data,logging, rules);
	    	
		} catch (Exception e) {
			logging.logToError("Could not load passive rules file - " + PASSIVERULESFILE + " - " + e.toString());
			return false;
		}
		
		logging.logToOutput("Loaded " + rules.size() + " rules"); 
    	
    	return true;
    }
    
    
    //
    // Constructor
    //
    MyScanCheck(MontoyaApi api, Logging logging)
    {
        this.api = api;
        this.logging = logging;
        this.rulesloaded = LoadRules();
    }  
 
    
    //
    // Active callback
    //
    @Override
    public AuditResult activeAudit(HttpRequestResponse baseRequestResponse, AuditInsertionPoint auditInsertionPoint)
    {
    	List<AuditIssue> auditIssueList = new ArrayList<AuditIssue>();
    	
    	if (baseRequestResponse == null || baseRequestResponse.response() == null || baseRequestResponse.response().body() == null) {
    		return auditResult(auditIssueList);
    	}
    	
    	
    		
		for(Rule myRule : rules)
		{
			try {
	    		String MyRegex = myRule.regex;
	    		String MyRegexDescription = myRule.name;
	    		//logging.logToOutput("Trying " + MyRegexDescription + " " + MyRegex + " on " + baseRequestResponse.response().bodyToString());
	    		//logging.logToOutput("Trying " + MyRegexDescription + " " + MyRegex);
	    		
	    		Pattern MyPattern = Pattern.compile(MyRegex, Pattern.DOTALL|Pattern.MULTILINE);
	    		Matcher MyMatcher = MyPattern.matcher(baseRequestResponse.response().bodyToString());
	    		
	    		//logging.logToOutput("Compiled " + MyRegexDescription + " " + MyRegex + " on " + baseRequestResponse.response().bodyToString());
	    		//logging.logToOutput("Compiled " + MyRegexDescription + " " + MyRegex);
	    		
	    		//
	    		// 
	    		//
	    		if(MyMatcher.matches()) {
	    			
	    			AuditIssue myIssue = auditIssue(
	                        "Sensitive Information Discovered",
	                        "Based on a regular expression search we have discovered  " + MyRegexDescription + " in the response body, confidence of rule is " + myRule.confidence,
	                        null,
	                        baseRequestResponse.request().url(),
	                        AuditIssueSeverity.INFORMATION,
	                        AuditIssueConfidence.CERTAIN,
	                        null,
	                        null,
	                        AuditIssueSeverity.INFORMATION,
	                        baseRequestResponse
	                );
					
					auditIssueList.add(myIssue);
	    		}
    		
			} catch (Exception PatternSyntaxException) {
				logging.logToError("Failed to compile regular expression due to syntax error " + PatternSyntaxException.toString());
	    	}
		}
    	
    	
    	
    	return auditResult(auditIssueList);
    }
    
    //
    // Passive callback
    //
    @Override
    public AuditResult passiveAudit(HttpRequestResponse baseRequestResponse)
    {
    	List<AuditIssue> auditIssueList = new ArrayList<AuditIssue>();
    	
    	if (baseRequestResponse == null || baseRequestResponse.response() == null || baseRequestResponse.response().body() == null) {
    		return auditResult(auditIssueList);
    	}
    	
		for(Rule myRule : rules)
		{
			try {
	    		String MyRegex = myRule.regex;
	    		String MyRegexDescription = myRule.name;
	    		//logging.logToOutput("Trying " + MyRegexDescription + " " + MyRegex + " on " + baseRequestResponse.response().bodyToString());
	    		//logging.logToOutput("Trying " + MyRegexDescription + " " + MyRegex);
	    		
	    		Pattern MyPattern = Pattern.compile(MyRegex, Pattern.DOTALL|Pattern.MULTILINE);
	    		Matcher MyMatcher = MyPattern.matcher(baseRequestResponse.response().bodyToString());
	    		
	    		//logging.logToOutput("Compiled " + MyRegexDescription + " " + MyRegex + " on " + baseRequestResponse.response().bodyToString());
	    		//logging.logToOutput("Compiled " + MyRegexDescription + " " + MyRegex);
	    		
	    		//
	    		// 
	    		//
	    		if(MyMatcher.matches()) {
	    				    		
	    			AuditIssue myIssue = auditIssue(
	                        "Sensitive Information Discovered",
	                        "Based on a regular expression search we have discovered  " + MyRegexDescription + " in the response body, confidence of rule is " + myRule.confidence,
	                        null,
	                        baseRequestResponse.request().url(),
	                        AuditIssueSeverity.INFORMATION,
	                        AuditIssueConfidence.CERTAIN,
	                        null,
	                        null,
	                        AuditIssueSeverity.INFORMATION,
	                        baseRequestResponse
	                );
					
					auditIssueList.add(myIssue);
	    			
	    		}
    		
			} catch (Exception PatternSyntaxException) {
	    		logging.logToError("Failed to compile regular expression due to syntax error " + PatternSyntaxException.toString());
	    	}
		}
    	
    	return auditResult(auditIssueList);
    }
    
    
    //
    // Consolidation callback
    //
    @Override
    public ConsolidationAction consolidateIssues(AuditIssue newIssue, AuditIssue existingIssue)
    {
    	return existingIssue.name().equals(newIssue.name()) ? KEEP_EXISTING : KEEP_BOTH;
    }

}

Rule.java

//
//
// PROTOTYPE Passive Sensitive Information Observer
//
// test with https://ginandjuice.shop/
//
// This has been tested with Burp Suite Pro to trial the concept.
//
// Ollie Whitehouse - @ollieatnowhere
//

package burp;

public class Rule {
	public String name;
	public String regex;
	public String confidence;
}