nvidia-smi --query-gpu="timestamp,pstate,temperature.gpu,utilization.gpu,utilization.memory,clocks_throttle_reasons.applications_clocks_setting,clocks_throttle_reasons.sw_power_cap,clocks_throttle_reasons.hw_slowdown,clocks_throttle_reasons.sync_boost,clocks_throttle_reasons.unknown,clocks.current.graphics,clocks.current.sm,clocks.current.memory…

nvidia-smi.log

timestamp, pstate, temperature.gpu, utilization.gpu [%], utilization.memory [%], clocks_throttle_reasons.applications_clocks_setting, clocks_throttle_reasons.sw_power_cap, clocks_throttle_reasons.hw_slowdown, clocks_throttle_reasons.sync_boost, clocks_throttle_reasons.unknown, clocks.current.graphics [MHz], clocks.current.sm [MHz], clocks.current.memory [MHz], clocks.current.video [MHz]
2017/03/20 19:58:43.613, P0, 67, 70 %, 32 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:44.615, P0, 67, 68 %, 32 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:45.616, P0, 67, 58 %, 28 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:46.618, P0, 67, 61 %, 30 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:47.619, P0, 67, 67 %, 32 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:48.621, P0, 67, 56 %, 27 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:49.623, P0, 68, 38 %, 18 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:50.624, P0, 68, 59 %, 26 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:51.626, P0, 68, 35 %, 17 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:52.628, P0, 68, 41 %, 19 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:53.629, P0, 67, 22 %, 11 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:54.631, P0, 68, 29 %, 14 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:55.633, P0, 68, 19 %, 8 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:56.635, P0, 67, 14 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:57.638, P0, 67, 14 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:58.641, P0, 67, 13 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:58:59.643, P0, 67, 21 %, 10 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:00.646, P0, 67, 16 %, 8 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:01.648, P0, 67, 17 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:02.650, P0, 67, 17 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:03.653, P0, 67, 16 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:04.655, P0, 67, 14 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:05.657, P0, 67, 16 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:06.660, P0, 67, 6 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:07.664, P0, 67, 8 %, 4 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:08.668, P0, 67, 4 %, 2 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:09.672, P0, 67, 8 %, 4 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:10.675, P0, 67, 9 %, 4 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:11.680, P0, 67, 8 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:12.684, P0, 67, 7 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:13.688, P0, 67, 10 %, 4 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:14.692, P2, 66, 12 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 46 MHz, 46 MHz, 3802 MHz, 113 MHz
2017/03/20 19:59:15.695, P2, 66, 11 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 46 MHz, 46 MHz, 3802 MHz, 113 MHz
2017/03/20 19:59:16.700, P5, 65, 30 %, 19 %, Not Active, Not Active, Not Active, Not Active, Not Active, 62 MHz, 62 MHz, 810 MHz, 167 MHz
2017/03/20 19:59:17.705, P5, 65, 24 %, 16 %, Not Active, Not Active, Not Active, Not Active, Not Active, 62 MHz, 62 MHz, 810 MHz, 167 MHz
2017/03/20 19:59:18.709, P5, 65, 35 %, 23 %, Not Active, Not Active, Not Active, Not Active, Not Active, 62 MHz, 62 MHz, 810 MHz, 167 MHz
2017/03/20 19:59:19.718, P5, 65, 42 %, 21 %, Not Active, Not Active, Not Active, Not Active, Not Active, 87 MHz, 87 MHz, 810 MHz, 24 MHz
2017/03/20 19:59:20.723, P5, 65, 26 %, 17 %, Not Active, Not Active, Not Active, Not Active, Not Active, 87 MHz, 87 MHz, 810 MHz, 24 MHz
2017/03/20 19:59:21.727, P5, 64, 32 %, 22 %, Not Active, Not Active, Not Active, Not Active, Not Active, 34 MHz, 34 MHz, 810 MHz, 113 MHz
2017/03/20 19:59:22.731, P5, 64, 27 %, 19 %, Not Active, Not Active, Not Active, Not Active, Not Active, 34 MHz, 34 MHz, 810 MHz, 113 MHz
2017/03/20 19:59:23.736, P5, 64, 29 %, 20 %, Not Active, Not Active, Not Active, Not Active, Not Active, 34 MHz, 34 MHz, 810 MHz, 113 MHz
2017/03/20 19:59:24.758, P5, 64, 28 %, 20 %, Not Active, Not Active, Not Active, Not Active, Not Active, 34 MHz, 34 MHz, 810 MHz, 113 MHz
2017/03/20 19:59:25.764, P0, 64, 14 %, 4 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:26.768, P0, 64, 19 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:27.771, P0, 64, 15 %, 4 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:28.776, P0, 64, 16 %, 4 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:29.780, P0, 64, 21 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:30.784, P0, 64, 35 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:31.788, P0, 64, 17 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:32.792, P0, 64, 18 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:33.796, P0, 63, 15 %, 4 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:34.801, P0, 63, 18 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:35.807, P0, 63, 18 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:36.811, P0, 63, 7 %, 2 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:37.815, P0, 63, 13 %, 2 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:38.818, P0, 63, 6 %, 2 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:39.822, P0, 63, 20 %, 6 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:40.838, P0, 63, 25 %, 7 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:41.842, P0, 63, 21 %, 6 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:42.846, P0, 63, 17 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:43.849, P0, 63, 27 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:44.854, P0, 63, 18 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:45.861, P0, 62, 18 %, 5 %, Not Active, Not Active, Not Active, Not Active, Not Active, 84 MHz, 84 MHz, 4006 MHz, 151 MHz
2017/03/20 19:59:46.886, P0, 64, 60 %, 30 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:47.888, P0, 65, 69 %, 29 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:48.890, P0, 66, 68 %, 30 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:49.892, P0, 66, 72 %, 31 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:50.893, P0, 65, 6 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:51.895, P0, 65, 6 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:52.896, P0, 65, 6 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz
2017/03/20 19:59:53.898, P0, 64, 5 %, 3 %, Not Active, Not Active, Not Active, Not Active, Not Active, 144 MHz, 144 MHz, 4006 MHz, 122 MHz