RSA encryption / decryption in salesforce apex

RSA.cls

@isTest
global without sharing class RSA {

    private Key key;
    // Hex digits
    private static final String DIGITS = '0123456789abcdef';
    private static final Decimal HEX_BASE = 16;

    public abstract class Key {
        private String modulus;
        public Key(String modulus) {
            this.modulus = modulus;
        }
    }
    
    public virtual class PublicKey extends Key {
        private String exponent;
        public PublicKey(String modulus, String exponent) {
            super(modulus);
            this.exponent = exponent;
        }
    }
    
    public class PrivateKey extends PublicKey {
        private String privateExponent;
        public PrivateKey(String modulus, String privateExponent) {
            super(modulus, null);
            this.privateExponent = privateExponent;
        }
        
        public PrivateKey(String modulus, String exponent, String privateExponent) {
            super(modulus, exponent);
            this.privateExponent = privateExponent;
        }
    }
    
    @isTest
    public static void Test()
    {
        //generate private key
        //openssl command: openssl genrsa -des3 -out private.pem 2048
        
        //openssl command: openssl rsa -in private.pem -noout -text
        //public key modulus (remove colons, remove 00 prefix & make into single line)
        //public key exp 0x10001 = 10001
        RSA.PublicKey pub = new RSA.PublicKey('f39dd44cceb6b25679e9318cabc7a679a7cd6caf5a96df59e52f74cc8f3d727a8c05d8a9067a700a93a728433bd1de65903d5ea2909761e06ff60bb980b5aaf48b3499d5b544448335e35474390f4873d04bb42f417e38fac333a5e7ffea5f67f4a9bc1df0a34a8b99c520a61711ffa9ab525c73ca13d1b9f44e35cc8bb7339bccd3a90115cffbc11349b216e7411d0f8d0f4770e3b77eaa8d62bb12f1ded9147d04f2aea39f09987b116cc47d996409718944e7f31dc416857eff79f9b2dd2c774fb70f9e0bea34b1538ee53a55dd92ee0dfa6e964112d6c4ff3770a44ecf06695932e6168ef7a64fa320a9d26f50b9b6109acfe148c74c40d37463719eef17'                                       
                                              ,'10001');    
                                              
        RSA r = new RSA(pub);
        
        String message = 'hello world hello world hello world hello world';
        String encrypted = r.encrypt(message);
        
        System.debug('Original Message: ' + message);
        System.debug('Encrypted Message: ' + encrypted);
        
        //openssl command: openssl rsa -in private.pem -noout -text
        //private key modulus (remove colons, remove 00 prefix & make into single line), same value as public key mod
        //private key privateExponent (remove colons & make into single line)
        RSA.PrivateKey pvt = new RSA.PrivateKey('f39dd44cceb6b25679e9318cabc7a679a7cd6caf5a96df59e52f74cc8f3d727a8c05d8a9067a700a93a728433bd1de65903d5ea2909761e06ff60bb980b5aaf48b3499d5b544448335e35474390f4873d04bb42f417e38fac333a5e7ffea5f67f4a9bc1df0a34a8b99c520a61711ffa9ab525c73ca13d1b9f44e35cc8bb7339bccd3a90115cffbc11349b216e7411d0f8d0f4770e3b77eaa8d62bb12f1ded9147d04f2aea39f09987b116cc47d996409718944e7f31dc416857eff79f9b2dd2c774fb70f9e0bea34b1538ee53a55dd92ee0dfa6e964112d6c4ff3770a44ecf06695932e6168ef7a64fa320a9d26f50b9b6109acfe148c74c40d37463719eef17'
                                               ,'00f2d322c5dc55a6b5239718d88a70dab2f05b8635d32a073ee77ec20113d5bfc1fec7e509b5775d2e6db6741f7004e4947f8d6c42c5b4dece834ad0acfa6a1a18de9873addc9c4b5e2ddc8655c27a45518b11aa6c5fef9c83f706081c93addda314f00a9e1d39e617f811d1553c31a8904a4031ff0831711ed5310fd6ee7c91668c6bfa61b1468cdaafb363dc6b9d529877499d067aa2a3d57435710a21543033925d403dc0b94ba603ea077036bd2422c63be75ec85f8be0b027be01c760f6a8d7a568fef22918b8a1d81cce012eb93a9d0ddf915c4749b6c5c9b58699001cecde0600414bf232c26c894377a359d97566c21f5e5f5783c43dd7fb170bf80c81');

        RSA r2 = new RSA(pvt);
        
        try
        {    
            String decrypted = r2.decrypt(encrypted );
            System.debug('Decrypted Message: ' + decrypted);
        }
        catch(Exception ex)
        {
            System.debug(ex.getMessage());
            System.debug(ex.getStackTraceString()); 
        }
    }

    public RSA(Key key) {
        this.key = key;
    }
    
    public String encrypt(String input) {
        PublicKey publicKey = (PublicKey)this.key;
        return modPowEncrypt(input, publicKey.modulus, publicKey.exponent);
    }
    
    public String decrypt(String input) {
        PrivateKey privateKey = (PrivateKey)this.key;
        return modPowDecrypt(input, privateKey.modulus, privateKey.privateExponent);
    }
    
    public String modPowEncrypt(String input, String modulus, String exponent) {
        
        //Blob mod = EncodingUtil.base64Decode(modulus);
        //Blob exp = EncodingUtil.base64Decode(exponent);
        
        Blob mod = Blob.valueOf(modulus);
        Blob exp = Blob.valueOf(exponent);
        
       	//System.debug('mod ' + hexToDecimal(EncodingUtil.convertToHex(mod)) );
        //System.debug('exp ' + hexToDecimal(EncodingUtil.convertToHex(exp)) );
        //System.debug(exp.toString());

        Blob pn = Blob.valueOf(input);
        
        // Pad password.nonce
        String temp = String.fromCharArray(pkcs1Pad2(input, mod.size()/2-1));
        
        //System.debug(pn.toString());
        //System.debug(mod.size());

        //Decimal modDec = hexToDecimal(EncodingUtil.convertToHex(mod));
        //Decimal expDec = hexToDecimal(EncodingUtil.convertToHex(exp));
        Decimal pnDec = hexToDecimal(EncodingUtil.convertToHex(pn));
        
        Decimal modDec = hexToDecimal(modulus);
        Decimal expDec = hexToDecimal(exponent);
        
        //System.debug(modDec.toPlainString());
        //System.debug(expDec.toPlainString());
        //System.debug(pnDec.toPlainString());

        // Calcluate padded^exp % mod and convert to hex
        Decimal result = modPow(pnDec, expDec, modDec);
        String hexResult = decimalToHex(result);
        // If length is uneven, add an extra 0
        if ((hexResult.length() & 1) == 1) {
            hexResult = '0' + hexResult;
        }

        //System.debug(hexResult);
        
        // Generate the data to be encrypted.
        Blob encodedData = EncodingUtil.convertFromHex(hexResult);
        
        return EncodingUtil.base64Encode(encodedData);
    }
    
    public String modPowDecrypt(String input, String modulus, String exponent ) {
        
        //Blob mod = EncodingUtil.base64Decode(modulus);
        //Blob exp = EncodingUtil.base64Decode(exponent);
        
        Blob mod = Blob.valueOf(modulus);
        Blob exp = Blob.valueOf(exponent);
        
       	//System.debug('mod ' + hexToDecimal(EncodingUtil.convertToHex(mod)) );
        //System.debug('exp ' + hexToDecimal(EncodingUtil.convertToHex(exp)) );
        //System.debug(exp.toString());
        
        Blob pn = EncodingUtil.base64Decode(input);
        
		//System.debug(pn.size());
        //System.debug(pn.toString());

        //Decimal modDec = hexToDecimal(EncodingUtil.convertToHex(mod));
        //Decimal expDec = hexToDecimal(EncodingUtil.convertToHex(exp));
        Decimal pnDec = hexToDecimal(EncodingUtil.convertToHex(pn));
        
        Decimal modDec = hexToDecimal(modulus);
        Decimal expDec = hexToDecimal(exponent);
        
        System.debug('mod ' + modDec.toPlainString());
        System.debug('exp ' + expDec.toPlainString());
        
        //System.debug(modDec.toPlainString());
        //System.debug(expDec.toPlainString());
        //System.debug(pnDec.toPlainString());

        // Calcluate padded^exp % mod and convert to hex
        Decimal result = modPow(pnDec, expDec, modDec);
        String hexResult = decimalToHex(result);
        // If length is uneven, add an extra 0
        if ((hexResult.length() & 1) == 1) {
            hexResult = '0' + hexResult;
        }

        String temp = blobToString(hexResult, 'UTF-8');

        return temp;
    }
    
    public static String blobToString(String hex, String inCharset)
    {
        //String hex = EncodingUtil.convertToHex(input);
        System.assertEquals(0, hex.length() & 1);
        final Integer bytesCount = hex.length() >> 1;
        String[] bytes = new String[bytesCount];
        for(Integer i = 0; i < bytesCount; ++i)
            bytes[i] =  hex.mid(i << 1, 2);
        
        for(Integer i = bytesCount-1; i >= 0; i--)
        {
            //System.debug(bytes[i]);
            
            if (bytes[i].equals('00'))
            {
                //System.debug('padding indicator');
                
                List<String> bytesTemp = new List<String>();
                
                for(Integer j = i; j < bytesCount; j++)
                {
                    bytesTemp.add(bytes[j]);
                }
                
                bytes = bytesTemp;
                
                break;
            }
        }
        
        //System.debug('%' + String.join(bytes, '%'));
        
        return EncodingUtil.urlDecode('%' + String.join(bytes, '%'), inCharset);
	}

    @testVisible
    private static Decimal hexToDecimal(String hex) {
        Decimal result = 0;
        integer length = hex.length();
        integer i = 0;
        while(i < length) {
            integer hexByte = DIGITS.indexOf(hex.substring(i, i + 1).toLowerCase());
            i++;
            result += hexByte * HEX_BASE.pow(length - i);
        }
        return result;
    }

    @testVisible
    private static String decimalToHex(Decimal d) {
        String hex = '';
        while (d > 0) {
            Decimal digit = modulus(d, HEX_BASE); // rightmost digit
            hex = DIGITS.substring(digit.intValue(), digit.intValue() + 1) + hex; // string concatenation
            d = d.divide(16, 0, RoundingMode.FLOOR);
        }
        return hex;
    }

    // base^exp % mod
    @testVisible
    private static Decimal modPow(Decimal base, Decimal exp, Decimal mod) {
        if (base < 1 || exp < 0 || mod < 1) {
            return -1;
        }

        Decimal result = 1;
        while (exp > 0) {
           if ((exp.longValue() & 1) == 1) {
               result = modulus((result * base), mod);
           }
           base = modulus((base * base), mod);
           exp = exp.divide(2, 0, RoundingMode.FLOOR);
        }
        return result;
    }

    // dividend % divisor
    @testVisible
    private static Decimal modulus(Decimal dividend, Decimal divisor) {
        Decimal d = dividend.divide(divisor, 0, RoundingMode.FLOOR);
        return dividend - (d * divisor);
    }

    // Pad using PKCS#1 v.2. See https://en.wikipedia.org/wiki/PKCS_1
    // s = String to pad
    // n = bytes to fill must be bigger than s.length()
    @testVisible
    private static List<integer> pkcs1Pad2(String s, integer n) {
        // Byte array
        List<integer> ba = new List<integer>();
        // Fill array with zeros to get the right size
        for(integer i = 0; i < n; i++) {
            ba.add(0);
        }
        integer i = s.length() - 1;
        while(i >= 0 && n > 0) {
            ba.set(--n, s.charAt(i--));
        }
        ba.set(--n, 0);
        while(n > 2) { // random non-zero pad
            // Since the array is converted to a string, choose integers that corresponds
            // to a proper char code see http://www.asciitable.com
            integer rnd = Math.round(Math.random() * (127 - 32) + 32);
            ba.set(--n, rnd);
        }
        ba.set(--n, 2);
        ba.set(--n, 0);
        return ba;
    }
}

Split encryption & decryption functions to demonstrate the flow of code.
Added test() method to demonstrate how it works including sample mod & exp with openssl commands in comments.
Major issue was the base64/unicode decoding that was causing issue in the original code for longer messages. It works for a small message.

When i ran this with 2048 size keys the resulting cipher text was 172 bytes which is invalid, it should be 256 bytes :/