Error from trying a docker image of graph_net.

woooo_buddy.txt

2018-10-22 21:44:44.331954: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-10-22 21:44:44.399573: E tensorflow/stream_executor/cuda/cuda_blas.cc:464] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED
2018-10-22 21:44:44.399625: W tensorflow/stream_executor/stream.cc:2089] attempting to perform BLAS operation using StreamExecutor without BLAS support
E1022 21:44:44.403764 140092477048576 tf_logging.py:105] Blas GEMM launch failed : a.shape=(3, 12), b.shape=(12, 10), m=3, n=10, k=12
	 [[{{node edge_block_1/mlp/linear_0/MatMul}} = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](edge_block_1/concat, edge_block/mlp/linear_0/w/read)]]

Caused by op u'edge_block_1/mlp/linear_0/MatMul', defined at:
  File "blocks_test.py", line 1103, in <module>
    tf.test.main()
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/test.py", line 64, in main
    return _googletest.main(argv)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/googletest.py", line 100, in main
    benchmark.benchmarks_main(true_main=main_wrapper)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/benchmark.py", line 344, in benchmarks_main
    true_main()
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/googletest.py", line 99, in main_wrapper
    return app.run(main=g_main, argv=args)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 125, in run
    _sys.exit(main(argv))
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/googletest.py", line 70, in g_main
    return unittest_main(argv=argv)
  File "/usr/lib/python2.7/unittest/main.py", line 95, in __init__
    self.runTests()
  File "/usr/lib/python2.7/unittest/main.py", line 232, in runTests
    self.result = testRunner.run(self.test)
  File "/usr/lib/python2.7/unittest/runner.py", line 151, in run
    test(result)
  File "/usr/lib/python2.7/unittest/suite.py", line 70, in __call__
    return self.run(*args, **kwds)
  File "/usr/lib/python2.7/unittest/suite.py", line 108, in run
    test(result)
  File "/usr/lib/python2.7/unittest/suite.py", line 70, in __call__
    return self.run(*args, **kwds)
  File "/usr/lib/python2.7/unittest/suite.py", line 108, in run
    test(result)
  File "/usr/lib/python2.7/unittest/case.py", line 393, in __call__
    return self.run(*args, **kwds)
  File "/usr/local/lib/python2.7/dist-packages/absl/third_party/unittest3_backport/case.py", line 162, in run
    testMethod()
  File "/usr/local/lib/python2.7/dist-packages/absl/testing/parameterized.py", line 258, in bound_param_test
    test_method(self, *testcase_params)
  File "blocks_test.py", line 1070, in test_dynamic_batch_sizes
    output = model(placeholders)
  File "/usr/local/lib/python2.7/dist-packages/sonnet/python/modules/base.py", line 389, in __call__
    outputs, subgraph_name_scope = self._template(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/template.py", line 360, in __call__
    return self._call_func(args, kwargs)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/template.py", line 311, in _call_func
    result = self._func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/sonnet/python/modules/base.py", line 246, in _build_wrapper
    output = self._build(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/graph_nets/blocks.py", line 454, in _build
    updated_edges = self._edge_model(collected_edges)
  File "/usr/local/lib/python2.7/dist-packages/sonnet/python/modules/base.py", line 389, in __call__
    outputs, subgraph_name_scope = self._template(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/template.py", line 360, in __call__
    return self._call_func(args, kwargs)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/template.py", line 311, in _call_func
    result = self._func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/sonnet/python/modules/base.py", line 246, in _build_wrapper
    output = self._build(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/sonnet/python/modules/nets/mlp.py", line 148, in _build
    net = self._layers[layer_id](net)
  File "/usr/local/lib/python2.7/dist-packages/sonnet/python/modules/base.py", line 389, in __call__
    outputs, subgraph_name_scope = self._template(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/template.py", line 360, in __call__
    return self._call_func(args, kwargs)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/template.py", line 311, in _call_func
    result = self._func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/sonnet/python/modules/base.py", line 246, in _build_wrapper
    output = self._build(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/sonnet/python/modules/basic.py", line 253, in _build
    outputs = tf.matmul(inputs, self._w)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/math_ops.py", line 2053, in matmul
    a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 4560, in mat_mul
    name=name)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
    return func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 3272, in create_op
    op_def=op_def)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1768, in __init__
    self._traceback = tf_stack.extract_stack()

InternalError (see above for traceback): Blas GEMM launch failed : a.shape=(3, 12), b.shape=(12, 10), m=3, n=10, k=12
	 [[{{node edge_block_1/mlp/linear_0/MatMul}} = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](edge_block_1/concat, edge_block/mlp/linear_0/w/read)]]

E2018-10-22 21:44:44.607614: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1490] Adding visible gpu devices: 0, 1, 2, 3, 4, 5, 6, 7
2018-10-22 21:44:44.608099: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-10-22 21:44:44.608116: I tensorflow/core/common_runtime/gpu/gpu_device.cc:977]      0 1 2 3 4 5 6 7 
2018-10-22 21:44:44.608126: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 0:   N Y Y Y N N N N 
2018-10-22 21:44:44.608135: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 1:   Y N Y Y N N N N 
2018-10-22 21:44:44.608141: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 2:   Y Y N Y N N N N 
2018-10-22 21:44:44.608148: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 3:   Y Y Y N N N N N 
2018-10-22 21:44:44.608156: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 4:   N N N N N Y Y Y 
2018-10-22 21:44:44.608165: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 5:   N N N N Y N Y Y 
2018-10-22 21:44:44.608172: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 6:   N N N N Y Y N Y 
2018-10-22 21:44:44.608180: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990] 7:   N N N N Y Y Y N 
2018-10-22 21:44:44.609583: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 3658 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:04:00.0, compute capability: 6.1)
2018-10-22 21:44:44.609979: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 3658 MB memory) -> physical GPU (device: 1, name: TITAN X (Pascal), pci bus id: 0000:05:00.0, compute capability: 6.1)
2018-10-22 21:44:44.610319: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:2 with 3658 MB memory) -> physical GPU (device: 2, name: TITAN X (Pascal), pci bus id: 0000:08:00.0, compute capability: 6.1)
2018-10-22 21:44:44.610609: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:3 with 3658 MB memory) -> physical GPU (device: 3, name: TITAN X (Pascal), pci bus id: 0000:09:00.0, compute capability: 6.1)
2018-10-22 21:44:44.610892: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:4 with 3658 MB memory) -> physical GPU (device: 4, name: TITAN X (Pascal), pci bus id: 0000:85:00.0, compute capability: 6.1)
2018-10-22 21:44:44.611213: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:5 with 3658 MB memory) -> physical GPU (device: 5, name: TITAN X (Pascal), pci bus id: 0000:86:00.0, compute capability: 6.1)
2018-10-22 21:44:44.611486: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:6 with 3658 MB memory) -> physical GPU (device: 6, name: TITAN X (Pascal), pci bus id: 0000:89:00.0, compute capability: 6.1)
2018-10-22 21:44:44.611736: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1103] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:7 with 3658 MB memory) -> physical GPU (device: 7, name: TITAN X (Pascal), pci bus id: 0000:8a:00.0, compute capability: 6.1)
2018-10-22 21:44:44.635483: I tensorflow/stream_executor/stream.cc:2038] [stream=0x91580030,impl=0x915800d0] did not wait for [stream=0x9157f9b0,impl=0x9157fa50]
2018-10-22 21:44:44.635532: I tensorflow/stream_executor/stream.cc:4986] [stream=0x91580030,impl=0x915800d0] did not memcpy host-to-device; source: 0x7f610ac00000
2018-10-22 21:44:44.635656: F tensorflow/core/common_runtime/gpu/gpu_util.cc:339] CPU->GPU Memcpy failed
*** Received signal 6 ***
*** BEGIN MANGLED STACK TRACE ***
/usr/local/lib/python2.7/dist-packages/tensorflow/python/../libtensorflow_framework.so(+0x6ce4eb)[0x7f69248c04eb]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x11390)[0x7f69d2121390]
/lib/x86_64-linux-gnu/libc.so.6(gsignal+0x38)[0x7f69d1d7b428]
/lib/x86_64-linux-gnu/libc.so.6(abort+0x16a)[0x7f69d1d7d02a]
/usr/local/lib/python2.7/dist-packages/tensorflow/python/_pywrap_tensorflow_internal.so(+0x4faf674)[0x7f692a14f674]
/usr/local/lib/python2.7/dist-packages/tensorflow/python/../libtensorflow_framework.so(+0x5fb022)[0x7f69247ed022]
/usr/local/lib/python2.7/dist-packages/tensorflow/python/../libtensorflow_framework.so(_ZN5Eigen26NonBlockingThreadPoolTemplIN10tensorflow6thread16EigenEnvironmentEE10WorkerLoopEi+0x21a)[0x7f692489a96a]
/usr/local/lib/python2.7/dist-packages/tensorflow/python/../libtensorflow_framework.so(_ZNSt17_Function_handlerIFvvEZN10tensorflow6thread16EigenEnvironment12CreateThreadESt8functionIS0_EEUlvE_E9_M_invokeERKSt9_Any_data+0x32)[0x7f6924899a12]
/usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80)[0x7f69910cac80]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba)[0x7f69d21176ba]
/lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7f69d1e4d41d]
*** END MANGLED STACK TRACE ***

*** Begin stack trace ***
	tensorflow::CurrentStackTrace()
	
	
	gsignal
	abort
	
	
	Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int)
	std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
	
	
	clone
*** End stack trace ***
Aborted (core dumped)