random_error.py

ncountered Exception while running query
Traceback (most recent call last):
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/xbb_tools-0.1-py3.7.egg/tools/utils.py", line 981, in run_dask_cudf_query
    results = query_func(client=client)
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/xbb_tools-0.1-py3.7.egg/tools/utils.py", line 308, in profiled
    result = func(*args, **kwargs)
  File "tpcx_bb_query_03.py", line 204, in main
    wcs_tstamp_min = get_wcs_minima(cli_args)
  File "tpcx_bb_query_03.py", line 39, in get_wcs_minima
    wcs_tstamp_min = wcs_df["tstamp"].min().compute()
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/dask/base.py", line 166, in compute
    (result,) = compute(self, traverse=False, **kwargs)
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/dask/base.py", line 437, in compute
    results = schedule(dsk, keys, **kwargs)
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/distributed/client.py", line 2632, in get
    results = self.gather(packed, asynchronous=asynchronous, direct=direct)
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/distributed/client.py", line 1931, in gather
    asynchronous=asynchronous,
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/distributed/client.py", line 780, in sync
    self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/distributed/utils.py", line 347, in sync
    raise exc.with_traceback(tb)
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/distributed/utils.py", line 331, in f
    result[0] = yield future
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
    value = future.result()
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/distributed/client.py", line 1790, in _gather
    raise exception.with_traceback(traceback)
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/dask/dataframe/io/parquet/core.py", line 268, in read_parquet_part
    dfs = [func(fs, rg, columns.copy(), index, **kwargs) for rg in part]
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/dask/dataframe/io/parquet/core.py", line 268, in <listcomp>
    dfs = [func(fs, rg, columns.copy(), index, **kwargs) for rg in part]
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/dask_cudf/io/parquet.py", line 55, in read_partition
    **kwargs.get("read", {}),
  File "/home/vjawa/conda/envs/cudf_may_1/lib/python3.7/site-packages/cudf/io/parquet.py", line 193, in read_parquet
    use_pandas_metadata=use_pandas_metadata,
  File "cudf/_lib/parquet.pyx", line 149, in cudf._lib.parquet.read_parquet
  File "cudf/_lib/parquet.pyx", line 185, in cudf._lib.parquet.read_parquet
RuntimeError: CUDA error encountered at: /conda/conda-bld/libcudf_1588347755069/work/cpp/src/io/utilities/hostdevice_vector.hpp:46: 717 cudaErrorInvalidAddressSpace operation not supported on global/shared address space