DeformableConvolution layer

README.md

Pytorch layer documentation: https://pytorch.org/vision/stable/ops.html#torchvision.ops.deform_conv2d

OpenVINO layer documentation: https://docs.openvinotoolkit.org/latest/openvino_docs_ops_convolution_DeformableConvolution_1.html

1. Create .onnx model with python3 export.py:

# export.py

import numpy as np
import torch
import torch.nn as nn
import torchvision


class DeformableConvolution(nn.Module):
    class DeformableConvolutionFunc(torch.autograd.Function):
        @staticmethod
        def symbolic(g, x, offset, weight, strides, pads, dilations):
            return g.op('DeformableConv2D', x, offset, weight, 
                        strides_i=strides,
                        pads_i=(pads[0], pads[0], pads[1], pads[1]),
                        dilations_i=dilations)

        @staticmethod
        def forward(self, x, offset, weight, strides, pads, dilations):
            y = torchvision.ops.deform_conv.deform_conv2d(x, offset=offset, weight=weight, 
                                                          stride=strides, padding=pads, dilation=dilations)
            return y


    def __init__(self, strides, pads, dilations):
        super(DeformableConvolution, self).__init__()
        self.strides = strides
        self.pads = pads
        self.dilations = dilations
        self.dconf2d = self.DeformableConvolutionFunc()
        self.pad = nn.ConstantPad2d((1, 1, 1, 1), 0)

    def forward(self, x, offset, weight):
        x = self.pad(x)
        offset = self.pad(offset)
        y = self.dconf2d.apply(x, offset, weight, self.strides, self.pads, self.dilations)
        y = y[:, :, 1:-1, 1:-1]
        return y


from torch import nn
class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()
        self.dconf2d = DeformableConvolution(strides=(1, 1), pads=(0, 0), dilations=(1, 1))

    def forward(self, x, offset, weight):
        y = self.dconf2d(x, offset, weight)
        return y


np.random.seed(324)
torch.manual_seed(32)

model = MyModel()
model.eval()

input = torch.rand(4, 3, 10, 10, dtype=torch.float32)
kh, kw = 3, 3
weight = torch.rand(5, 3, kh, kw, dtype=torch.float32)
offset = torch.rand(4, 2 * kh * kw, 8, 8, dtype=torch.float32)

with torch.no_grad():
    torch.onnx.export(model, (input, offset, weight), 'model.onnx',
                      input_names=['input', 'offset', 'weight'],
                      output_names=['output'],
                      operator_export_type=torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK,
                      opset_version=12)

ref = model(input, offset, weight)
np.save('inp', input.detach().numpy())
np.save('offset', offset.detach().numpy())
np.save('weight', weight.detach().numpy())
np.save('ref', ref.detach().numpy())

2. Create OpenVINO IR:

python3 /opt/intel/openvino_2021/deployment_tools/model_optimizer/mo.py --input_model model.onnx

3. Check the result with python3 compare.py:

# compare.py

import numpy as np
from openvino.inference_engine import IECore

ref = np.load('ref.npy')

inputs = {}
inputs['input'] = np.load('inp.npy')
inputs['offset'] = np.load('offset.npy')
inputs['weight'] = np.load('weight.npy')

ie = IECore()
net = ie.read_network('model.xml', 'model.bin')
exec_net = ie.load_network(net, 'CPU')

out = exec_net.infer(inputs)
out = next(iter(out.values()))

maxdiff = np.max(np.abs(ref - out))
print('Reference range: [{}, {}]'.format(np.min(ref), np.max(ref)))
print('Maximal difference:', maxdiff)
if maxdiff > 1e-5:
    exit(1)

Expected difference is 2.861023e-06

Thank you for your help!! 😄