gistfile1.txt

⠠ [00:00:14] [############################>-----------] 45/64 (6s) - Chunk 45/64, Loss: NaN
thread 'main' panicked at /home/erikg/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cubecl-cuda-0.4.0/src/compute/server.rs:115:22:
called `Result::unwrap()` on an `Err` value: DriverError(CUDA_ERROR_ILLEGAL_ADDRESS, "an illegal memory access was encountered")
stack backtrace:
   0:     0x6449b217ba3a - <std::sys::backtrace::BacktraceLock::print::DisplayBacktrace as core::fmt::Display>::fmt::he089f96442833f67
   1:     0x6449b21a2f43 - core::fmt::write::h2f210ed4c94745cb
   2:     0x6449b2178cb3 - std::io::Write::write_fmt::h7de08171ab770fb2
   3:     0x6449b217b882 - std::sys::backtrace::BacktraceLock::print::h810fbd31421329e6
   4:     0x6449b217c8f0 - std::panicking::default_hook::{{closure}}::hbaad47ed9dc6356d
   5:     0x6449b217c6d0 - std::panicking::default_hook::h24e207139139d40a
   6:     0x6449b217d052 - std::panicking::rust_panic_with_hook::ha9131beeb2ddc506
   7:     0x6449b217cdfa - std::panicking::begin_panic_handler::{{closure}}::h1bba0eaeb6da506f
   8:     0x6449b217bf39 - std::sys::backtrace::__rust_end_short_backtrace::h1d1ca3eade483f4c
   9:     0x6449b217ca8d - rust_begin_unwind
  10:     0x6449b1c8edd0 - core::panicking::panic_fmt::h896a0727a1a943f9
  11:     0x6449b1c8f1a6 - core::result::unwrap_failed::h1b5ed8541c7bebd6
  12:     0x6449b2104f00 - <cubecl_cuda::compute::server::CudaServer as cubecl_runtime::server::ComputeServer>::read::h6f652febcfb717f1
  13:     0x6449b1e1ed63 - cubecl_runtime::client::ComputeClient<Server,Channel>::read_one_async::{{closure}}::h37c8fbfddc600c4d
  14:     0x6449b1e2a2e5 - burn_jit::ops::float_ops::<impl burn_tensor::tensor::ops::tensor::FloatTensorOps<burn_jit::backend::JitBackend<R,F,I,BT>> for burn_jit::backend::JitBackend<R,F,I,BT>>::float_into_data::{{closure}}::h730ed2f770ed1758
  15:     0x6449b1ddfc14 - <burn_tensor::tensor::api::kind::Float as burn_tensor::tensor::api::base::BasicOps<B>>::into_data_async::{{closure}}::hf1202a9202725189
  16:     0x6449b1de0675 - burn_tensor::tensor::api::base::Tensor<B,_,K>::into_data_async::{{closure}}::h586d107a5d4e7a73
  17:     0x6449b1e141be - futures_lite::future::block_on::h9588842a57bae99f
  18:     0x6449b1ff8ed7 - grufinity::tbptt::train_with_tbptt::hc58bccd15e5f30bd
  19:     0x6449b1e3f8e2 - tbptt_train::main::h70e24f4542b45397
  20:     0x6449b1fcdbc3 - std::sys::backtrace::__rust_begin_short_backtrace::hf3c8b0d418fc9ca3
  21:     0x6449b1fcdbb9 - std::rt::lang_start::{{closure}}::h9d43fb764b3ad63d
  22:     0x6449b21744d7 - std::rt::lang_start_internal::hbb268f70c879621d
  23:     0x6449b1e40345 - main
  24:     0x7fe1cc42a1ca - __libc_start_call_main
                               at ./csu/../sysdeps/nptl/libc_start_call_main.h:58:16
  25:     0x7fe1cc42a28b - __libc_start_main_impl
                               at ./csu/../csu/libc-start.c:360:3
  26:     0x6449b1c8f3f5 - _start
  27:                0x0 - <unknown>