Skip to content

Instantly share code, notes, and snippets.

@VibhuJawa

VibhuJawa/memory_shootup_cudf.ipynb

Created October 7, 2019 23:36
Show Gist options
  • Save VibhuJawa/829931018e0831e2baf5af083d8d44a4 to your computer and use it in GitHub Desktop.
Save VibhuJawa/829931018e0831e2baf5af083d8d44a4 to your computer and use it in GitHub Desktop.
Download ZIP
xgb_memory_shootup_cudf.ipynb
Display the source blob
Display the rendered blob
Raw
memory_shootup_cudf.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import cudf\n",
"import xgboost\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"df = cudf.DataFrame({str(i):np.ones(20_000_000,dtype=np.int32) for i in range(0,40)})"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Mon Oct 7 16:35:02 2019 \n",
"+-----------------------------------------------------------------------------+\n",
"| NVIDIA-SMI 410.48 Driver Version: 410.48 |\n",
"|-------------------------------+----------------------+----------------------+\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n",
"|===============================+======================+======================|\n",
"| 0 Tesla P100-SXM2... On | 00000000:06:00.0 Off | 0 |\n",
"| N/A 35C P0 48W / 300W | 3477MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 1 Tesla P100-SXM2... On | 00000000:07:00.0 Off | 0 |\n",
"| N/A 34C P0 34W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 2 Tesla P100-SXM2... On | 00000000:0A:00.0 Off | 0 |\n",
"| N/A 30C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 3 Tesla P100-SXM2... On | 00000000:0B:00.0 Off | 0 |\n",
"| N/A 31C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 4 Tesla P100-SXM2... On | 00000000:85:00.0 Off | 0 |\n",
"| N/A 32C P0 33W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 5 Tesla P100-SXM2... On | 00000000:86:00.0 Off | 0 |\n",
"| N/A 31C P0 34W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 6 Tesla P100-SXM2... On | 00000000:89:00.0 Off | 0 |\n",
"| N/A 31C P0 35W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 7 Tesla P100-SXM2... On | 00000000:8A:00.0 Off | 0 |\n",
"| N/A 32C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
" \n",
"+-----------------------------------------------------------------------------+\n",
"| Processes: GPU Memory |\n",
"| GPU PID Type Process name Usage |\n",
"|=============================================================================|\n",
"| 0 5902 C .../vjawa/anaconda3/envs/RAPIDS/bin/python 3467MiB |\n",
"+-----------------------------------------------------------------------------+\n"
]
}
],
"source": [
"!nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"dmat = xgboost.DMatrix(df)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Mon Oct 7 16:35:05 2019 \n",
"+-----------------------------------------------------------------------------+\n",
"| NVIDIA-SMI 410.48 Driver Version: 410.48 |\n",
"|-------------------------------+----------------------+----------------------+\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n",
"|===============================+======================+======================|\n",
"| 0 Tesla P100-SXM2... On | 00000000:06:00.0 Off | 0 |\n",
"| N/A 35C P0 48W / 300W | 9745MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 1 Tesla P100-SXM2... On | 00000000:07:00.0 Off | 0 |\n",
"| N/A 34C P0 34W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 2 Tesla P100-SXM2... On | 00000000:0A:00.0 Off | 0 |\n",
"| N/A 30C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 3 Tesla P100-SXM2... On | 00000000:0B:00.0 Off | 0 |\n",
"| N/A 31C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 4 Tesla P100-SXM2... On | 00000000:85:00.0 Off | 0 |\n",
"| N/A 32C P0 33W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 5 Tesla P100-SXM2... On | 00000000:86:00.0 Off | 0 |\n",
"| N/A 31C P0 34W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 6 Tesla P100-SXM2... On | 00000000:89:00.0 Off | 0 |\n",
"| N/A 31C P0 35W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 7 Tesla P100-SXM2... On | 00000000:8A:00.0 Off | 0 |\n",
"| N/A 32C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
" \n",
"+-----------------------------------------------------------------------------+\n",
"| Processes: GPU Memory |\n",
"| GPU PID Type Process name Usage |\n",
"|=============================================================================|\n",
"| 0 5902 C .../vjawa/anaconda3/envs/RAPIDS/bin/python 9735MiB |\n",
"+-----------------------------------------------------------------------------+\n"
]
}
],
"source": [
"!nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"del df"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Mon Oct 7 16:35:12 2019 \n",
"+-----------------------------------------------------------------------------+\n",
"| NVIDIA-SMI 410.48 Driver Version: 410.48 |\n",
"|-------------------------------+----------------------+----------------------+\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n",
"|===============================+======================+======================|\n",
"| 0 Tesla P100-SXM2... On | 00000000:06:00.0 Off | 0 |\n",
"| N/A 35C P0 48W / 300W | 6625MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 1 Tesla P100-SXM2... On | 00000000:07:00.0 Off | 0 |\n",
"| N/A 34C P0 35W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 2 Tesla P100-SXM2... On | 00000000:0A:00.0 Off | 0 |\n",
"| N/A 30C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 3 Tesla P100-SXM2... On | 00000000:0B:00.0 Off | 0 |\n",
"| N/A 31C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 4 Tesla P100-SXM2... On | 00000000:85:00.0 Off | 0 |\n",
"| N/A 32C P0 33W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 5 Tesla P100-SXM2... On | 00000000:86:00.0 Off | 0 |\n",
"| N/A 30C P0 34W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 6 Tesla P100-SXM2... On | 00000000:89:00.0 Off | 0 |\n",
"| N/A 31C P0 35W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 7 Tesla P100-SXM2... On | 00000000:8A:00.0 Off | 0 |\n",
"| N/A 32C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
" \n",
"+-----------------------------------------------------------------------------+\n",
"| Processes: GPU Memory |\n",
"| GPU PID Type Process name Usage |\n",
"|=============================================================================|\n",
"| 0 5902 C .../vjawa/anaconda3/envs/RAPIDS/bin/python 6615MiB |\n",
"+-----------------------------------------------------------------------------+\n"
]
}
],
"source": [
"!nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"del dmat"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Mon Oct 7 16:35:30 2019 \n",
"+-----------------------------------------------------------------------------+\n",
"| NVIDIA-SMI 410.48 Driver Version: 410.48 |\n",
"|-------------------------------+----------------------+----------------------+\n",
"| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
"| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n",
"|===============================+======================+======================|\n",
"| 0 Tesla P100-SXM2... On | 00000000:06:00.0 Off | 0 |\n",
"| N/A 35C P0 48W / 300W | 367MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 1 Tesla P100-SXM2... On | 00000000:07:00.0 Off | 0 |\n",
"| N/A 34C P0 34W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 2 Tesla P100-SXM2... On | 00000000:0A:00.0 Off | 0 |\n",
"| N/A 30C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 3 Tesla P100-SXM2... On | 00000000:0B:00.0 Off | 0 |\n",
"| N/A 31C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 4 Tesla P100-SXM2... On | 00000000:85:00.0 Off | 0 |\n",
"| N/A 32C P0 33W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 5 Tesla P100-SXM2... On | 00000000:86:00.0 Off | 0 |\n",
"| N/A 31C P0 34W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 6 Tesla P100-SXM2... On | 00000000:89:00.0 Off | 0 |\n",
"| N/A 31C P0 35W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
"| 7 Tesla P100-SXM2... On | 00000000:8A:00.0 Off | 0 |\n",
"| N/A 32C P0 32W / 300W | 10MiB / 16280MiB | 0% Default |\n",
"+-------------------------------+----------------------+----------------------+\n",
" \n",
"+-----------------------------------------------------------------------------+\n",
"| Processes: GPU Memory |\n",
"| GPU PID Type Process name Usage |\n",
"|=============================================================================|\n",
"| 0 5902 C .../vjawa/anaconda3/envs/RAPIDS/bin/python 357MiB |\n",
"+-----------------------------------------------------------------------------+\n"
]
}
],
"source": [
"!nvidia-smi"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment