gistfile1.txt

java -Xmx10g -cp target/dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar org.deeplearning4j.examples.rnn.GravesLSTMCharModellingExample 2>&1 |tee GPU-benchmark-0.4.0-3.txt
o.n.n.NativeOps - Number of threads used for linear algebra 32
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [0]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [1]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [2]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [3]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [4]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [5]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [6]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.a.c.i.BasicContextPool - Creating new stream for thread: [1], device: [7]...
o.n.j.c.CudaAffinityManager - Mapping thread [1] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [27] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [28] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [29] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [30] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [31] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [32] to device [0], out of [8] devices...
o.n.j.h.i.CudaZeroHandler - Creating bucketID: 5
o.n.j.h.i.CudaZeroHandler - Creating bucketID: 0
o.n.j.h.i.CudaZeroHandler - Creating bucketID: 3
o.n.j.h.i.CudaZeroHandler - Creating bucketID: 4
o.n.j.h.i.CudaZeroHandler - Creating bucketID: 1
o.n.j.h.i.CudaZeroHandler - Creating bucketID: 2
Using existing text file at /tmp/Shakespeare.txt
o.n.j.c.CudaAffinityManager - Mapping thread [144] to device [1], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [146] to device [2], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [123] to device [3], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [125] to device [4], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [134] to device [5], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [137] to device [6], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [119] to device [7], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [124] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [145] to device [1], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [140] to device [2], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [127] to device [3], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [149] to device [4], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [147] to device [5], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [143] to device [6], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [141] to device [7], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [136] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [138] to device [1], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [120] to device [2], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [122] to device [3], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [126] to device [4], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [128] to device [5], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [129] to device [6], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [133] to device [7], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [148] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [139] to device [1], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [131] to device [2], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [132] to device [3], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [142] to device [4], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [121] to device [5], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [135] to device [6], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [118] to device [7], out of [8] devices...
o.n.j.c.CudaAffinityManager - Mapping thread [130] to device [0], out of [8] devices...
o.d.s.i.p.ParameterAveragingTrainingMaster - Starting training of split 1 of 1. workerMiniBatchSize=8, averagingFreq=3, Configured for 32 workers
[Stage 4:=======>                                                 (4 + 28) / 32]o.n.j.c.CudaAffinityManager - Manually mapping thread [398] to device [5], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [397] to device [1], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [396] to device [4], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [399] to device [5], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [400] to device [6], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [402] to device [7], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [403] to device [3], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [404] to device [7], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [405] to device [2], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [401] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [406] to device [4], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [407] to device [5], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [408] to device [3], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [409] to device [1], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [410] to device [6], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [411] to device [4], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [412] to device [6], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [413] to device [5], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [414] to device [1], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [415] to device [6], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [416] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [417] to device [2], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [419] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [418] to device [7], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [421] to device [1], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [422] to device [7], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [433] to device [0], out of [8] devices...
o.n.j.c.CudaAffinityManager - Manually mapping thread [434] to device [4], out of [8] devices...
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" 
Failed on [140503481981904] -> [35868684800], size: [307692], direction: [0], result: [77]
Failed on [140505362010848] -> [35639787520], size: [307692], direction: [0], result: [77]
Failed on [140504420758064] -> [35642192384], size: [307692], direction: [0], result: [77]
Failed on [140505159116224] -> [35870781952], size: [307692], direction: [0], result: [77]
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" 
Failed on [35918708736] -> [180751826944], size: [2239508], direction: [1], result: [77]
o.a.s.e.Executor - Exception in task 12.0 in stage 4.0 (TID 140)
java.lang.IllegalStateException: MemcpyAsync relocate H2D failed: [35918708736] -> [180751826944]
	at org.nd4j.jita.handler.impl.CudaZeroHandler.relocate(CudaZeroHandler.java:366) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.jita.handler.impl.CudaZeroHandler.getDevicePointer(CudaZeroHandler.java:733) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.jita.allocator.impl.AtomicAllocator.getPointer(AtomicAllocator.java:256) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.invoke(JCudaExecutioner.java:1001) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.exec(JCudaExecutioner.java:552) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
o.a.s.s.TaskSetManager - Task 12 in stage 4.0 failed 1 times; aborting job
Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 12 in stage 4.0 failed 1 times, most recent failure: Lost task 12.0 in stage 4.0 (TID 140, localhost): java.lang.IllegalStateException: MemcpyAsync relocate H2D failed: [35918708736] -> [180751826944]
	at org.nd4j.jita.handler.impl.CudaZeroHandler.relocate(CudaZeroHandler.java:366)
	at org.nd4j.jita.handler.impl.CudaZeroHandler.getDevicePointer(CudaZeroHandler.java:733)
	at org.nd4j.jita.allocator.impl.AtomicAllocator.getPointer(AtomicAllocator.java:256)
	at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.invoke(JCudaExecutioner.java:1001)
	at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.exec(JCudaExecutioner.java:552)
	at org.nd4j.linalg.api.ndarray.BaseNDArray.assign(BaseNDArray.java:2943)
	at org.nd4j.linalg.factory.BaseNDArrayFactory.ones(BaseNDArrayFactory.java:1247)
	at org.nd4j.linalg.factory.BaseNDArrayFactory.ones(BaseNDArrayFactory.java:972)
	at org.nd4j.linalg.factory.Nd4j.ones(Nd4j.java:4504)
	at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.initMask(MultiLayerNetwork.java:1879)
	at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.init(MultiLayerNetwork.java:416)
	at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingWorker.getInitialModel(ParameterAveragingTrainingWorker.java:44)
	at org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap.call(ExecuteWorkerFlatMap.java:68)
	at org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap.call(ExecuteWorkerFlatMap.java:26)
	at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:156)
	at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:156)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:706)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:706)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
	at org.apache.spark.scheduler.Task.run(Task.scala:88)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1280)
	at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1268)
	at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1267)
	at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1267)
	at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697)
	at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697)
	at scala.Option.foreach(Option.scala:257)
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1493)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1455)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1444)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1813)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:1933)CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" 

CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" 
	at org.apache.spark.rdd.RDD$$anonfun$aggregate$1.apply(RDD.scala:1078)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)Failed on [140504555050896] -> [35534496768], size: [307692], direction: [0], result: [77]

	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:306)
	at org.apache.spark.rdd.RDD.aggregate(RDD.scala:1071)
	at org.apache.spark.api.java.JavaRDDLike$class.aggregate(JavaRDDLike.scala:416)
	at org.apache.spark.api.java.AbstractJavaRDDLike.aggregate(JavaRDDLike.scala:47)
	at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster.processResults(ParameterAveragingTrainingMaster.java:407)
	at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster.doIteration(ParameterAveragingTrainingMaster.java:339)
	at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster.executeTraining(ParameterAveragingTrainingMaster.java:181)
	at org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer.fit(SparkDl4jMultiLayer.java:201)
	at org.deeplearning4j.examples.rnn.GravesLSTMCharModellingExample.main(GravesLSTMCharModellingExample.java:115)
Caused by: java.lang.IllegalStateException: MemcpyAsync relocate H2D failed: [35918708736] -> [180751826944]
	at org.nd4j.jita.handler.impl.CudaZeroHandler.relocate(CudaZeroHandler.java:366)
	at org.nd4j.jita.handler.impl.CudaZeroHandler.getDevicePointer(CudaZeroHandler.java:733)
	at org.nd4j.jita.allocator.impl.AtomicAllocator.getPointer(AtomicAllocator.java:256)
	at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.invoke(JCudaExecutioner.java:1001)
	at org.nd4j.linalg.jcublas.ops.executioner.JCudaExecutioner.exec(JCudaExecutioner.java:552)
	at org.nd4j.linalg.api.ndarray.BaseNDArray.assign(BaseNDArray.java:2943)
	at org.nd4j.linalg.factory.BaseNDArrayFactory.ones(BaseNDArrayFactory.java:1247)
	at org.nd4j.linalg.factory.BaseNDArrayFactory.ones(BaseNDArrayFactory.java:972)
	at org.nd4j.linalg.factory.Nd4j.ones(Nd4j.java:4504)
o.n.j.h.i.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [1], requested bytes: [2239508]
	at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.initMask(MultiLayerNetwork.java:1879)
	at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.init(MultiLayerNetwork.java:416)
	at org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingWorker.getInitialModel(ParameterAveragingTrainingWorker.java:44)
	at org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap.call(ExecuteWorkerFlatMap.java:68)
	at org.deeplearning4j.spark.api.worker.ExecuteWorkerFlatMap.call(ExecuteWorkerFlatMap.java:26)
	at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:156)
	at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$4$1.apply(JavaRDDLike.scala:156)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:706)
	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$17.apply(RDD.scala:706)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
o.n.j.h.i.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [1], requested bytes: [2239508]
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
	at org.apache.spark.scheduler.Task.run(Task.scala:88)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
o.n.j.h.i.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [5], requested bytes: [2239508]
o.n.j.h.i.CudaZeroHandler - Out of [DEVICE] memory, host memory will be used instead: deviceId: [5], requested bytes: [2239508]
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result" 
Failed on [140504352848288] -> [35692831744], size: [307692], direction: [0], result: [77]
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:2831 code=77(<unknown>) "cudaStreamSynchronize(*stream)" 
o.a.s.e.Executor - Exception in task 26.0 in stage 4.0 (TID 154)
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504352848288] -> [35692831744]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:68) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_91]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_91]
	at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1058) ~[na:1.8.0_91]
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504352848288] -> [35692831744]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:737) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.buffer.BaseDataBuffer.read(BaseDataBuffer.java:1129) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.read(BaseNDArrayProxy.java:75) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:66) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na]
Caused by: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504352848288] -> [35692831744]
	at org.nd4j.jita.handler.impl.CudaZeroHandler.memcpyAsync(CudaZeroHandler.java:510) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.jita.allocator.impl.AtomicAllocator.memcpyAsync(AtomicAllocator.java:774) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.set(BaseCudaDataBuffer.java:324) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.setData(BaseCudaDataBuffer.java:371) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:728) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
o.a.s.e.Executor - Exception in task 22.0 in stage 4.0 (TID 150)
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140505362010848] -> [35639787520]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:68) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_91]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_91]
	at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1058) ~[na:1.8.0_91]
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140505362010848] -> [35639787520]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:737) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.buffer.BaseDataBuffer.read(BaseDataBuffer.java:1129) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.read(BaseNDArrayProxy.java:75) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:66) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na]
Caused by: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140505362010848] -> [35639787520]
	at org.nd4j.jita.handler.impl.CudaZeroHandler.memcpyAsync(CudaZeroHandler.java:510) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.jita.allocator.impl.AtomicAllocator.memcpyAsync(AtomicAllocator.java:774) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.set(BaseCudaDataBuffer.java:324) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.setData(BaseCudaDataBuffer.java:371) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:728) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
o.a.s.e.Executor - Exception in task 17.0 in stage 4.0 (TID 145)
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504420758064] -> [35642192384]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:68) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_91]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_91]
	at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1058) ~[na:1.8.0_91]
Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504420758064] -> [35642192384]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:737) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.buffer.BaseDataBuffer.read(BaseDataBuffer.java:1129) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.read(BaseNDArrayProxy.java:75) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.api.ndarray.BaseNDArrayProxy.readObject(BaseNDArrayProxy.java:66) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[na:na]
Caused by: java.lang.IllegalStateException: MemcpyAsync H2H failed: [140504420758064] -> [35642192384]
	at org.nd4j.jita.handler.impl.CudaZeroHandler.memcpyAsync(CudaZeroHandler.java:510) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.jita.allocator.impl.AtomicAllocator.memcpyAsync(AtomicAllocator.java:774) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.set(BaseCudaDataBuffer.java:324) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.setData(BaseCudaDataBuffer.java:371) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
	at org.nd4j.linalg.jcublas.buffer.BaseCudaDataBuffer.read(BaseCudaDataBuffer.java:728) ~[dl4j-spark-cdh5-examples-1.0-SNAPSHOT.jar:na]
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3921 code=77(<unknown>) "cudaStreamSynchronize(*pStream)" 
CUDA error at /skymind/libnd4j/blas/cuda/NativeOps.cu:3947 code=77(<unknown>) "result"