Skip to content

Instantly share code, notes, and snippets.

@gerwang

gerwang/align_diligentmv_psnerf.ipynb

Created September 26, 2022 04:55
Show Gist options
  • Save gerwang/d4d03eeea4a496ae5494ad075ac009d7 to your computer and use it in GitHub Desktop.
Save gerwang/d4d03eeea4a496ae5494ad075ac009d7 to your computer and use it in GitHub Desktop.
Download ZIP
Obtain the transformation matrix from data in PSNeRF's space to the original DiLiGent-MV space
Display the source blob
Display the rendered blob
Raw
align_diligentmv_psnerf.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 51,
"id": "980b811a-6958-4f73-be31-ea672b3c30c7",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import json\n",
"from scipy import io as sio\n",
"import trimesh"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "776f098c-776a-4354-afea-af039923e953",
"metadata": {},
"outputs": [],
"source": [
"import numpy\n",
"\n",
"\n",
"def affine_matrix_from_points(v0, v1, scale=True):\n",
" \"\"\"Return affine transform matrix to register two point sets.\n",
" v0 and v1 are shape (ndims, \\*) arrays of at least ndims non-homogeneous\n",
" coordinates, where ndims is the dimensionality of the coordinate space.\n",
" If shear is False, a similarity transformation matrix is returned.\n",
" If also scale is False, a rigid/Euclidean transformation matrix\n",
" is returned.\n",
" By default the algorithm by Hartley and Zissermann [15] is used.\n",
" If usesvd is True, similarity and Euclidean transformation matrices\n",
" are calculated by minimizing the weighted sum of squared deviations\n",
" (RMSD) according to the algorithm by Kabsch [8].\n",
" Otherwise, and if ndims is 3, the quaternion based algorithm by Horn [9]\n",
" is used, which is slower when using this Python implementation.\n",
" The returned matrix performs rotation, translation and uniform scaling\n",
" (if specified).\n",
" >>> numpy.set_printoptions(precision=5, suppress=True)\n",
" >>> v0 = [[0, 1031, 1031, 0], [0, 0, 1600, 1600]]\n",
" >>> v1 = [[675, 826, 826, 677], [55, 52, 281, 277]]\n",
" >>> affine_matrix_from_points(v0, v1)\n",
" array([[ 0.14549, 0.00062, 675.50008],\n",
" [ 0.00048, 0.14094, 53.24971],\n",
" [ 0. , 0. , 1. ]])\n",
" >>> T = translation_matrix(numpy.random.random(3)-0.5)\n",
" >>> R = random_rotation_matrix(numpy.random.random(3))\n",
" >>> S = scale_matrix(numpy.random.random())\n",
" >>> M = concatenate_matrices(T, R, S)\n",
" >>> v0 = (numpy.random.rand(4, 100) - 0.5) * 20\n",
" >>> v0[3] = 1\n",
" >>> v1 = numpy.dot(M, v0)\n",
" >>> v0[:3] += numpy.random.normal(0, 1e-8, 300).reshape(3, -1)\n",
" >>> M = affine_matrix_from_points(v0[:3], v1[:3])\n",
" >>> numpy.allclose(v1, numpy.dot(M, v0))\n",
" True\n",
" More examples in superimposition_matrix()\n",
" \"\"\"\n",
" v0 = numpy.array(v0, dtype=numpy.float64, copy=True)\n",
" v1 = numpy.array(v1, dtype=numpy.float64, copy=True)\n",
"\n",
" ndims = v0.shape[0]\n",
" if ndims < 2 or v0.shape[1] < ndims or v0.shape != v1.shape:\n",
" raise ValueError(\"input arrays are of wrong shape or type\")\n",
"\n",
" # move centroids to origin\n",
" t0 = -numpy.mean(v0, axis=1)\n",
" M0 = numpy.identity(ndims + 1)\n",
" M0[:ndims, ndims] = t0\n",
" v0 += t0.reshape(ndims, 1)\n",
" t1 = -numpy.mean(v1, axis=1)\n",
" M1 = numpy.identity(ndims + 1)\n",
" M1[:ndims, ndims] = t1\n",
" v1 += t1.reshape(ndims, 1)\n",
"\n",
" # Rigid transformation via SVD of covariance matrix\n",
" u, s, vh = numpy.linalg.svd(numpy.dot(v1, v0.T))\n",
" # rotation matrix from SVD orthonormal bases\n",
" R = numpy.dot(u, vh)\n",
" if numpy.linalg.det(R) < 0.0:\n",
" # R does not constitute right handed system\n",
" R -= numpy.outer(u[:, ndims - 1], vh[ndims - 1, :] * 2.0)\n",
" s[-1] *= -1.0\n",
" # homogeneous transformation matrix\n",
" M = numpy.identity(ndims + 1)\n",
" M[:ndims, :ndims] = R\n",
"\n",
" if scale:\n",
" # Affine transformation; scale is ratio of RMS deviations from centroid\n",
" v0 *= v0\n",
" v1 *= v1\n",
" M[:ndims, :ndims] *= numpy.sqrt(numpy.sum(v1) / numpy.sum(v0))\n",
"\n",
" # move centroids back\n",
" M = numpy.dot(numpy.linalg.inv(M1), numpy.dot(M, M0))\n",
" M /= M[ndims, ndims]\n",
" return M"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "5036a472-0f42-4804-9d73-d955ca3d9cca",
"metadata": {},
"outputs": [],
"source": [
"json_path = '/path/to/psnerf/data/bear/params.json'\n",
"mat_path = '/path/to/DiLiGenT-MV/mvpmsData/bearPNG/Calib_Results.mat'"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "26e469e2-b0ec-40ef-b564-6536fdc1420c",
"metadata": {},
"outputs": [],
"source": [
"np.set_printoptions(precision = 3)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "e60c07d7-45f7-4d78-9280-b05626c2f3e1",
"metadata": {},
"outputs": [],
"source": [
"json_dict = json.load(open(json_path))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "2809907b-cd4b-42b8-995a-f3f75a237e63",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"dict_keys(['obj_name', 'n_view', 'view_train', 'view_test', 'K', 'pose_c2w', 'light_is_same', 'light_direction', 'gt_normal_world', 'imhw', 'light_intensity'])"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"json_dict.keys()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "707bba04-4c14-477f-92dd-bf4eb00626ac",
"metadata": {},
"outputs": [],
"source": [
"mat_dict = sio.loadmat(mat_path)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "ce158145-5d19-4a69-a31d-6d3f17932cb4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"dict_keys(['__header__', '__version__', '__globals__', 'KK', 'Rc_1', 'Tc_1', 'Rc_2', 'Tc_2', 'Rc_3', 'Tc_3', 'Rc_4', 'Tc_4', 'Rc_5', 'Tc_5', 'Rc_6', 'Tc_6', 'Rc_7', 'Tc_7', 'Rc_8', 'Tc_8', 'Rc_9', 'Tc_9', 'Rc_10', 'Tc_10', 'Rc_11', 'Tc_11', 'Rc_12', 'Tc_12', 'Rc_13', 'Tc_13', 'Rc_14', 'Tc_14', 'Rc_15', 'Tc_15', 'Rc_16', 'Tc_16', 'Rc_17', 'Tc_17', 'Rc_18', 'Tc_18', 'Rc_19', 'Tc_19', 'Rc_20', 'Tc_20'])"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mat_dict.keys()"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "29542962-2150-4224-b44d-b85803b83071",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 7.881e-03, -4.447e-01, 8.957e-01],\n",
" [ 1.000e+00, 8.589e-05, -8.756e-03],\n",
" [ 3.817e-03, 8.957e-01, 4.447e-01]])"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.linalg.inv(mat_dict['Rc_1']) @ np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]])"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "70405ac0-2526-442e-8d42-5d45dfa1a1b6",
"metadata": {},
"outputs": [],
"source": [
"tmp = np.vstack([np.hstack([mat_dict['Rc_1'], mat_dict['Tc_1']]), np.array([[0, 0, 0, 1]])])"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "a8f39367-6a9b-476b-b85d-c5df829c4080",
"metadata": {},
"outputs": [],
"source": [
"tmp2 = np.linalg.inv(tmp) @ np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]])"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "b569fc10-298b-4794-9aaa-6231c6430b03",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 7.881e-03, -4.447e-01, 8.957e-01, 1.448e+03],\n",
" [ 1.000e+00, 8.589e-05, -8.756e-03, 7.055e+01],\n",
" [ 3.817e-03, 8.957e-01, 4.447e-01, 7.102e+02],\n",
" [ 0.000e+00, 0.000e+00, 0.000e+00, 1.000e+00]])"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tmp2"
]
},
{
"cell_type": "markdown",
"id": "3affc08b-4b19-46bc-acae-82e71f51bf11",
"metadata": {},
"source": [
"The matlab file stores w2c camera, in OpenGL format; Whilist the json file stores c2w camera, in OpenCV format."
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "50052a3a-75df-4af3-8adb-e74f530c6485",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(3, 1)"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mat_dict['Tc_1'].shape"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "5c1d94dd-870a-47be-b434-e5b81fe3df4d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"20"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(json_dict['pose_c2w'])"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "4320fada-c726-4809-bce0-0168d6719d41",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 7.881e-03, -4.447e-01, 8.957e-01, 2.785e+01],\n",
" [ 1.000e+00, 8.589e-05, -8.756e-03, -2.439e-01],\n",
" [ 3.817e-03, 8.957e-01, 4.447e-01, 1.349e+01],\n",
" [ 0.000e+00, 0.000e+00, 0.000e+00, 1.000e+00]])"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.array(json_dict['pose_c2w'][0])"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "993c73cb-7314-4ac2-8e38-bd380c33f282",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 27.847, -0.244, 13.494],\n",
" [ 26.223, -9.362, 13.532],\n",
" [ 21.921, -17.231, 13.562],\n",
" [ 15.725, -23.098, 13.576],\n",
" [ 7.178, -27.083, 13.564],\n",
" [ -0.931, -28.077, 13.528],\n",
" [ -8.471, -26.855, 13.47 ],\n",
" [-16.238, -23.109, 13.386],\n",
" [-22.47 , -17.205, 13.362],\n",
" [-26.97 , -8.763, 13.357],\n",
" [-28.39 , 0.574, 13.383],\n",
" [-27.115, 8.452, 13.427],\n",
" [-22.69 , 17.066, 13.47 ],\n",
" [-16.392, 23.147, 13.498],\n",
" [ -8.09 , 27.122, 13.532],\n",
" [ 1.003, 28.198, 13.542],\n",
" [ 9.681, 26.405, 13.519],\n",
" [ 18.231, 21.274, 13.477],\n",
" [ 23.4 , 15.265, 13.466],\n",
" [ 26.894, 7.358, 13.474]])"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"json_campos = []\n",
"for x in json_dict['pose_c2w']:\n",
" json_campos.append(np.array(x)[:3, 3])\n",
"json_campos = np.stack(json_campos)\n",
"json_campos"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "7541201a-9d6f-44f8-9561-7ab43fc166a9",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 1448.494, 70.547, 710.202],\n",
" [ 1368.949, -376.224, 712.081],\n",
" [ 1158.146, -761.802, 713.549],\n",
" [ 854.537, -1049.326, 714.233],\n",
" [ 435.712, -1244.561, 713.633],\n",
" [ 38.388, -1293.29 , 711.86 ],\n",
" [ -331.093, -1233.389, 709.028],\n",
" [ -711.67 , -1049.831, 704.908],\n",
" [-1017.024, -760.565, 703.748],\n",
" [-1237.516, -346.883, 703.499],\n",
" [-1307.091, 110.638, 704.75 ],\n",
" [-1244.64 , 496.654, 706.914],\n",
" [-1027.799, 918.743, 709.024],\n",
" [ -719.232, 1216.712, 710.392],\n",
" [ -312.386, 1411.464, 712.058],\n",
" [ 133.17 , 1464.208, 712.541],\n",
" [ 558.389, 1376.324, 711.452],\n",
" [ 977.305, 1124.932, 709.374],\n",
" [ 1230.589, 830.503, 708.837],\n",
" [ 1401.82 , 443.062, 709.218]])"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mat_campos = []\n",
"for i in range(len(json_dict['pose_c2w'])):\n",
" tmp = np.vstack([np.hstack([mat_dict[f'Rc_{i + 1}'], mat_dict[f'Tc_{i + 1}']]), np.array([[0, 0, 0, 1]])])\n",
" tmp = np.linalg.inv(tmp) @ np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]])\n",
" mat_campos.append(tmp[:3, 3])\n",
"mat_campos = np.stack(mat_campos)\n",
"mat_campos"
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "d32cd8a0-411d-438c-b97e-bd52710a10b7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 4.900e+01, 5.039e-08, 2.348e-07, 8.400e+01],\n",
" [-5.039e-08, 4.900e+01, 2.363e-07, 8.250e+01],\n",
" [-2.348e-07, -2.363e-07, 4.900e+01, 4.900e+01],\n",
" [ 0.000e+00, 0.000e+00, 0.000e+00, 1.000e+00]])"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"M = affine_matrix_from_points(json_campos.T, mat_campos.T)\n",
"M"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "2cc3cb14-b7aa-4ccc-875d-ca0cef2c2c14",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[ 1448.494, 70.547, 710.202],\n",
" [ 1368.949, -376.224, 712.081],\n",
" [ 1158.146, -761.802, 713.549],\n",
" [ 854.537, -1049.326, 714.233],\n",
" [ 435.712, -1244.561, 713.633],\n",
" [ 38.388, -1293.29 , 711.86 ],\n",
" [ -331.093, -1233.389, 709.028],\n",
" [ -711.67 , -1049.831, 704.908],\n",
" [-1017.024, -760.565, 703.748],\n",
" [-1237.516, -346.883, 703.499],\n",
" [-1307.091, 110.638, 704.751],\n",
" [-1244.64 , 496.654, 706.914],\n",
" [-1027.799, 918.743, 709.024],\n",
" [ -719.232, 1216.712, 710.392],\n",
" [ -312.386, 1411.464, 712.058],\n",
" [ 133.17 , 1464.208, 712.541],\n",
" [ 558.389, 1376.324, 711.452],\n",
" [ 977.305, 1124.932, 709.374],\n",
" [ 1230.589, 830.503, 708.837],\n",
" [ 1401.82 , 443.062, 709.218]])"
]
},
"execution_count": 43,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(M[:3, :3] @ json_campos.T + M[:3, 3:]).T"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "e9a8a8bc-8ba0-4ef7-b289-c36ba14f85a8",
"metadata": {},
"outputs": [],
"source": [
"mesh = trimesh.load('/path/to/psnerf/mesh/bear.ply')"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "4048677f-86b4-4236-bb77-4ff1e83b817b",
"metadata": {},
"outputs": [],
"source": [
"mesh.vertices = (M[:3, :3] @ mesh.vertices.T + M[:3, 3:]).T"
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "d42c2eee-ae25-40cf-97db-b8f03fffd554",
"metadata": {},
"outputs": [],
"source": [
"mesh.export('/path/to/psnerf/mesh/bear_world.ply');"
]
},
{
"cell_type": "code",
"execution_count": 46,
"id": "98b82a99-9949-4fa9-9426-8dc9a7d0ca69",
"metadata": {},
"outputs": [],
"source": [
"obj_names = ['bear', 'buddha', 'cow', 'pot2', 'reading']"
]
},
{
"cell_type": "code",
"execution_count": 47,
"id": "40de7cd1-9ebd-49a5-867d-ac7d65b23e82",
"metadata": {},
"outputs": [],
"source": [
"json_template_path = '/path/to/psnerf/data/{}/params.json'\n",
"mat_template_path = '/path/to/DiLiGenT-MV/mvpmsData/{}PNG/Calib_Results.mat'\n",
"output_template_path = '/path/to/psnerf/data/{}/scale_mat.npy'"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "5a340ef5-a323-4e5e-b08f-4cfa4c81b1e0",
"metadata": {},
"outputs": [],
"source": [
"def process(obj_name):\n",
" json_path = json_template_path.format(obj_name)\n",
" mat_path = mat_template_path.format(obj_name)\n",
" output_path = output_template_path.format(obj_name)\n",
" \n",
" json_dict = json.load(open(json_path))\n",
" json_campos = []\n",
" for x in json_dict['pose_c2w']:\n",
" json_campos.append(np.array(x)[:3, 3])\n",
" json_campos = np.stack(json_campos)\n",
" \n",
" mat_dict = sio.loadmat(mat_path)\n",
" mat_campos = []\n",
" for i in range(len(json_dict['pose_c2w'])):\n",
" tmp = np.vstack([np.hstack([mat_dict[f'Rc_{i + 1}'], mat_dict[f'Tc_{i + 1}']]), np.array([[0, 0, 0, 1]])])\n",
" tmp = np.linalg.inv(tmp) @ np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]])\n",
" mat_campos.append(tmp[:3, 3])\n",
" mat_campos = np.stack(mat_campos)\n",
" M = affine_matrix_from_points(json_campos.T, mat_campos.T)\n",
" np.save(output_path, M)"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "3c19b781-91cf-4a61-a797-808ed9237e11",
"metadata": {},
"outputs": [],
"source": [
"for obj_name in obj_names:\n",
" process(obj_name)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment