sample for using a yubihsm2 with python requests

engine.py

try:
  from OpenSSL._util import (
      ffi as _ffi,
      lib as O,
      lib as S
      )
  pyopenssl = True

  import sys

  null = _ffi.NULL
  # the engine loaded here has to have an ABI matching that of the openssl that
  # is used by cryptography, that means it's probably openssl 1.1
  p11 = b"/tmp/openssl-p11/prefix/lib/engines-1.1/pkcs11.so"

except ImportError:
  pyopenssl = False
  import ctypes

  O = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libcrypto.so")
  S = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libssl.so")

  null = None
  p11 = b"/usr/lib/ssl/engines/libpkcs11.so"

O.ENGINE_load_dynamic()
e = O.ENGINE_by_id(b"dynamic")
print(e)
print(O.ENGINE_ctrl_cmd_string(e, b"SO_PATH", p11, 0))
print(O.ENGINE_ctrl_cmd_string(e, b"LOAD", null, 0))
# where to find the yubihsm_pkcs11.so
print(O.ENGINE_ctrl_cmd_string(e, b"MODULE_PATH", b"yubihsm_pkcs11.so", 0))
# authenticate to the YubiHSM (id+password)
print(O.ENGINE_ctrl_cmd_string(e, b"PIN", b"0001password", 0))
print(O.ENGINE_init(e))
# load a key with id 0x0001
key = O.ENGINE_load_private_key(e, b"0:0001", null, null)

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.ssl_ import create_urllib3_context
from requests.packages.urllib3.poolmanager import PoolManager

class YubihsmAdapter(HTTPAdapter):
  def init_poolmanager(self, connections, maxsize, block=False, *args, **kwargs):
    context = create_urllib3_context()
    if(pyopenssl):
      # if pyopenssl is in use we can find the SSL_CTX in a semi-clean way
      ctx = context._ctx._context
    else:
      # if the builtin ssl module is in use we have to jump blindly to the
      # third element in the struct and rely on cpython returning the adress
      # with id() on an object.
      ctx = ctypes.c_void_p.from_address(id(context) + ctypes.sizeof(ctypes.c_void_p) * 2)

    # point the SSL_CTX to a certificate matching our key
    print(S.SSL_CTX_use_certificate_file(ctx, b"cert.pem", 1))
    # and load the key into the SSL_CTX
    print(S.SSL_CTX_use_PrivateKey(ctx, key))
    kwargs['ssl_context'] = context
    self.poolmanager = PoolManager(
        num_pools=connections, maxsize=maxsize,
        block=block, *args, **kwargs)

s = requests.Session()
# the URL "mounted" has to be longer than https:// and match whatever it is
# we're going to use for our requests
s.mount("https://127.0.0.1", YubihsmAdapter())
print(s.get("https://127.0.0.1:8443", verify=False))

Hi @klali thank yo u for sharing. I'm trying to use Openssl engine in my project. Following your code I was able to enable engine and load private key to engine (by calling O.ENGINE_load_private_key()). But I have problem to find the correct ctx to pass to SSL_CTX_use_PrivateKey(). I'm using twisted here is my code

from twisted.internet.ssl import ContextFactory, DefaultOpenSSLContextFactory
from OpenSSL import SSL, crypto

class EngineOpenSSLContextFactory(ContextFactory):

    _context = None

    def __init__(self, privateKeyFileName, certificateFileName,
                 sslmethod=SSL.SSLv23_METHOD, _contextFactory=SSL.Context):
        """
        @param privateKeyFileName: Name of a file containing a private key
        @param certificateFileName: Name of a file containing a certificate
        @param sslmethod: The SSL method to use
        """
        self.privateKeyFileName = privateKeyFileName
        self.certificateFileName = certificateFileName
        self.sslmethod = sslmethod
        self._contextFactory = _contextFactory
        self.cacheContext()

    def cacheContext(self):
        self._context = self._contextFactory(self.sslmethod)
        self.init_ssl_engine('tpm2tss')

    def init_ssl_engine(self, engine):
        _lib = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libcrypto.so")
        _lib2 = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libssl.so")

        _lib.ENGINE_load_builtin_engines()
        e = _lib.ENGINE_by_id(b"tpm2tss")
        if e == None:
            raise ValueError('Cannot find engine {}'.format(engine))

        if not _lib.ENGINE_init(e):
            _lib.ENGINE_free(e)
            raise Exception('Cannot initialize engine {}'.format(engine))

        u = _lib.UI_OpenSSL()
        key = _lib.ENGINE_load_private_key(e, b"/home/dwang3/Desktop/tls-tpm/tpm-gen-cert/tpm-client-priv.tss", u, None)

        ctx = self._context
        print(_lib2.SSL_CTX_use_PrivateKey(ctx, key))

Python3 reports "print(_lib2.SSL_CTX_use_PrivateKey(ctx, key))
ctypes.ArgumentError: argument 1: <class 'TypeError'>: Don't know how to convert parameter 1" Do you have any idea how to pass ctx in this case? Thank you again and appreciate your help!

Twisted looks like it's using pyopenssl, looking at what seems to be stored in twisted that should maybe be self._context._context.

Thank you so much @klali. I tried that as your suggested. But I still get the same error:

ctx = self._context._context
print(_lib2.SSL_CTX_use_PrivateKey(ctx, key))
print(_lib2.SSL_CTX_use_certificate_file(ctx, b"/home/dwang3/Desktop/tls-tpm/tpm-gen-cert/tpm-client-cert.pem", 1))

print(_lib2.SSL_CTX_use_PrivateKey(ctx, key))
ctypes.ArgumentError: argument 1: <class 'TypeError'>: Don't know how to convert parameter 1

I'm actually using ctypes not the OpenSSL._util. Here is my setup:

import ctypes
_lib = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libcrypto.so")
_lib2 = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libssl.so")

I assume it should not matter but I do not know if it makes a difference?

Second question, When I import OpenSSL._util as your code. Python reports error as following do you know why? Did I do something wrong?

_lib.ENGINE_load_builtin_engines()
AttributeError: module 'lib' has no attribute 'ENGINE_load_builtin_engines'
or 
AttributeError: module 'lib' has no attribute 'ENGINE_load_dynamic'

Thank you so much for your help!

This was useful to me, wanted to share a cleaner (IMO) way of accessing the context object with ssl:

You have this:

ctx = ctypes.cast(id(context) + ctypes.sizeof(ctypes.c_void_p) * 2, ctypes.POINTER(ctypes.c_void_p)).contents

This also works:

from ctypes import c_void_p, sizeof
ctx = c_void_p.from_address(id(context) + sizeof(c_void_p) * 2)

from_address uses the value at the given address to create the c_void_p.

Second question, When I import OpenSSL._util as your code. Python reports error as following do you know why? Did I do something wrong?

_lib.ENGINE_load_builtin_engines()
AttributeError: module 'lib' has no attribute 'ENGINE_load_builtin_engines'
or 
AttributeError: module 'lib' has no attribute 'ENGINE_load_dynamic'

You probably need to use cryptography==2.5 as cryptography==2.6.1 remove a lot of "unused" function.
Commit: