Skip to content

Instantly share code, notes, and snippets.

@rabernat

rabernat/zarr_parquet_benchmark.ipynb

Created April 3, 2025 16:18
Show Gist options
  • Save rabernat/ae0e1ee4b64f932717c7c6378cd2fbb3 to your computer and use it in GitHub Desktop.
Save rabernat/ae0e1ee4b64f932717c7c6378cd2fbb3 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
zarr_parquet_benchmark.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "0fe188b5-3a99-4c34-bc2a-882f2ba7ac75",
"metadata": {},
"source": [
"# Zarr + Xarray vs. DuckDB and Parquet\n",
"\n",
"## Initialize Connections\n",
"\n",
"For these benchmarks, we exclude the initial scanning of the file, which has a one-time cost compared to subsequent queries.\n",
"The assumption here is that the same dataset object will be used repeatedly for multiple queries."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "566e1a28-d28d-4e31-9c07-82470c4d159c",
"metadata": {},
"outputs": [],
"source": [
"import xarray as xr\n",
"import icechunk as ic\n",
"import duckdb\n",
"import zarr"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "ffc76af4-8342-4205-91f5-1d78e1c35bcd",
"metadata": {},
"outputs": [],
"source": [
"duckdb.sql(\"INSTALL spatial; LOAD spatial;\")\n",
"duckdb.sql(\"INSTALL httpfs; LOAD httpfs;\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "4da89a5d-e56b-4d13-a478-6023260c78d1",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<donfig.config_obj.ConfigSet at 0x7f7d964de630>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# make sure Zarr uses all threads on this EC2 instance\n",
"# (DuckDB does this automatically)\n",
"zarr.config.set(\n",
" {\n",
" 'threading.max_workers': 16,\n",
" 'async.concurrency': 32\n",
" }\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "dc4260f1-a5dc-4a5d-83a5-00f1e9cce875",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/srv/conda/envs/notebook/lib/python3.12/site-packages/numcodecs/zarr3.py:170: UserWarning: Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.\n",
" super().__init__(**codec_config)\n",
"/srv/conda/envs/notebook/lib/python3.12/site-packages/numcodecs/zarr3.py:170: UserWarning: Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.\n",
" super().__init__(**codec_config)\n",
"/srv/conda/envs/notebook/lib/python3.12/site-packages/numcodecs/zarr3.py:170: UserWarning: Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.\n",
" super().__init__(**codec_config)\n",
"/srv/conda/envs/notebook/lib/python3.12/site-packages/numcodecs/zarr3.py:170: UserWarning: Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.\n",
" super().__init__(**codec_config)\n",
"/srv/conda/envs/notebook/lib/python3.12/site-packages/numcodecs/zarr3.py:170: UserWarning: Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.\n",
" super().__init__(**codec_config)\n"
]
},
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=\"dark\"],\n",
"html[data-theme=\"dark\"],\n",
"body[data-theme=\"dark\"],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1f1f1f;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 0 20px 0 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: inline-block;\n",
" opacity: 0;\n",
" height: 0;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:focus + label {\n",
" border: 2px solid var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: \"►\";\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: \"▼\";\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: \"(\";\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: \")\";\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: \",\";\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt; Size: 4GB\n",
"Dimensions: (longitude: 1440, latitude: 721, valid_time: 209)\n",
"Coordinates:\n",
" * latitude (latitude) float64 6kB 90.0 89.75 89.5 ... -89.5 -89.75 -90.0\n",
" * valid_time (valid_time) datetime64[ns] 2kB 2024-05-13 ... 2024-05-29\n",
" time datetime64[ns] 8B ...\n",
" * longitude (longitude) float64 12kB 0.0 0.25 0.5 0.75 ... -0.75 -0.5 -0.25\n",
"Data variables:\n",
" gust (longitude, latitude, valid_time) float32 868MB ...\n",
" t2m (longitude, latitude, valid_time) float32 868MB ...\n",
" tcc (longitude, latitude, valid_time) float32 868MB ...\n",
" prate (longitude, latitude, valid_time) float32 868MB ...\n",
" r2 (longitude, latitude, valid_time) float32 868MB ...\n",
"Attributes:\n",
" description: GFS data ingested for forecasting demo</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-703a8f3d-6927-4ce4-bd82-9496973719b7' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-703a8f3d-6927-4ce4-bd82-9496973719b7' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span class='xr-has-index'>longitude</span>: 1440</li><li><span class='xr-has-index'>latitude</span>: 721</li><li><span class='xr-has-index'>valid_time</span>: 209</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-63e01a9a-e833-45d0-9627-0b7f8839f09d' class='xr-section-summary-in' type='checkbox' checked><label for='section-63e01a9a-e833-45d0-9627-0b7f8839f09d' class='xr-section-summary' >Coordinates: <span>(4)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>latitude</span></div><div class='xr-var-dims'>(latitude)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>90.0 89.75 89.5 ... -89.75 -90.0</div><input id='attrs-dbe45a4b-c046-4f9a-b044-787b9d46099e' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-dbe45a4b-c046-4f9a-b044-787b9d46099e' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-b55cb635-1d45-46f8-b47a-f259141dbcf0' class='xr-var-data-in' type='checkbox'><label for='data-b55cb635-1d45-46f8-b47a-f259141dbcf0' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>Y</dd><dt><span>standard_name :</span></dt><dd>latitude</dd><dt><span>units :</span></dt><dd>degrees_north</dd></dl></div><div class='xr-var-data'><pre>array([ 90. , 89.75, 89.5 , ..., -89.5 , -89.75, -90. ])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>valid_time</span></div><div class='xr-var-dims'>(valid_time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2024-05-13 ... 2024-05-29</div><input id='attrs-728b4841-0fac-4525-ae97-92f121c5e340' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-728b4841-0fac-4525-ae97-92f121c5e340' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-0c4d0dbf-e100-4571-8cc1-6a5e5fa1cd17' class='xr-var-data-in' type='checkbox'><label for='data-0c4d0dbf-e100-4571-8cc1-6a5e5fa1cd17' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([&#x27;2024-05-13T00:00:00.000000000&#x27;, &#x27;2024-05-13T01:00:00.000000000&#x27;,\n",
" &#x27;2024-05-13T02:00:00.000000000&#x27;, ..., &#x27;2024-05-28T18:00:00.000000000&#x27;,\n",
" &#x27;2024-05-28T21:00:00.000000000&#x27;, &#x27;2024-05-29T00:00:00.000000000&#x27;],\n",
" dtype=&#x27;datetime64[ns]&#x27;)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>time</span></div><div class='xr-var-dims'>()</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-68ae794d-7e6b-4ad3-90b8-8311ef0f17d8' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-68ae794d-7e6b-4ad3-90b8-8311ef0f17d8' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-d687db8e-af33-4a34-9bd7-6898b8ff1118' class='xr-var-data-in' type='checkbox'><label for='data-d687db8e-af33-4a34-9bd7-6898b8ff1118' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>axis :</span></dt><dd>T</dd><dt><span>standard_name :</span></dt><dd>forecast_reference_time</dd></dl></div><div class='xr-var-data'><pre>[1 values with dtype=datetime64[ns]]</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>longitude</span></div><div class='xr-var-dims'>(longitude)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>0.0 0.25 0.5 ... -0.75 -0.5 -0.25</div><input id='attrs-0923ed40-e5c0-42f8-b86c-2a594618099d' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-0923ed40-e5c0-42f8-b86c-2a594618099d' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-79a94e50-9dc1-4c7e-a1d1-41e113f0d438' class='xr-var-data-in' type='checkbox'><label for='data-79a94e50-9dc1-4c7e-a1d1-41e113f0d438' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([ 0. , 0.25, 0.5 , ..., -0.75, -0.5 , -0.25])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-964e8b12-9341-4434-ab3a-b2877da37631' class='xr-section-summary-in' type='checkbox' checked><label for='section-964e8b12-9341-4434-ab3a-b2877da37631' class='xr-section-summary' >Data variables: <span>(5)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>gust</span></div><div class='xr-var-dims'>(longitude, latitude, valid_time)</div><div class='xr-var-dtype'>float32</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-01081520-0718-48b1-ad9d-2b5ead039e93' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-01081520-0718-48b1-ad9d-2b5ead039e93' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-acd72490-b531-45a5-8ab4-249537f70ec9' class='xr-var-data-in' type='checkbox'><label for='data-acd72490-b531-45a5-8ab4-249537f70ec9' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>GRIB_NV :</span></dt><dd>0</dd><dt><span>GRIB_Nx :</span></dt><dd>1440</dd><dt><span>GRIB_Ny :</span></dt><dd>721</dd><dt><span>GRIB_cfName :</span></dt><dd>unknown</dd><dt><span>GRIB_cfVarName :</span></dt><dd>gust</dd><dt><span>GRIB_dataType :</span></dt><dd>fc</dd><dt><span>GRIB_gridDefinitionDescription :</span></dt><dd>Latitude/longitude. Also called equidistant cylindrical, or Plate Carree</dd><dt><span>GRIB_gridType :</span></dt><dd>regular_ll</dd><dt><span>GRIB_iDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_iScansNegatively :</span></dt><dd>0</dd><dt><span>GRIB_jDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_jPointsAreConsecutive :</span></dt><dd>0</dd><dt><span>GRIB_jScansPositively :</span></dt><dd>0</dd><dt><span>GRIB_latitudeOfFirstGridPointInDegrees :</span></dt><dd>90.0</dd><dt><span>GRIB_latitudeOfLastGridPointInDegrees :</span></dt><dd>-90.0</dd><dt><span>GRIB_longitudeOfFirstGridPointInDegrees :</span></dt><dd>0.0</dd><dt><span>GRIB_longitudeOfLastGridPointInDegrees :</span></dt><dd>359.75</dd><dt><span>GRIB_missingValue :</span></dt><dd>3.4028234663852886e+38</dd><dt><span>GRIB_name :</span></dt><dd>Wind speed (gust)</dd><dt><span>GRIB_numberOfPoints :</span></dt><dd>1038240</dd><dt><span>GRIB_paramId :</span></dt><dd>260065</dd><dt><span>GRIB_parameterName :</span></dt><dd>Wind speed (gust)</dd><dt><span>GRIB_parameterUnits :</span></dt><dd>m s-1</dd><dt><span>GRIB_shortName :</span></dt><dd>gust</dd><dt><span>GRIB_stepRange :</span></dt><dd>0</dd><dt><span>GRIB_stepType :</span></dt><dd>instant</dd><dt><span>GRIB_stepUnits :</span></dt><dd>1</dd><dt><span>GRIB_typeOfLevel :</span></dt><dd>surface</dd><dt><span>GRIB_units :</span></dt><dd>m s**-1</dd><dt><span>grid_mapping :</span></dt><dd>gribfile_projection</dd><dt><span>long_name :</span></dt><dd>Wind speed (gust)</dd><dt><span>standard_name :</span></dt><dd>unknown</dd><dt><span>units :</span></dt><dd>m s**-1</dd></dl></div><div class='xr-var-data'><pre>[216992160 values with dtype=float32]</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>t2m</span></div><div class='xr-var-dims'>(longitude, latitude, valid_time)</div><div class='xr-var-dtype'>float32</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-16981176-8c29-49fe-a2f7-82bd467463af' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-16981176-8c29-49fe-a2f7-82bd467463af' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-4befec5e-305a-4489-9802-95abf6229e01' class='xr-var-data-in' type='checkbox'><label for='data-4befec5e-305a-4489-9802-95abf6229e01' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>GRIB_NV :</span></dt><dd>0</dd><dt><span>GRIB_Nx :</span></dt><dd>1440</dd><dt><span>GRIB_Ny :</span></dt><dd>721</dd><dt><span>GRIB_cfName :</span></dt><dd>air_temperature</dd><dt><span>GRIB_cfVarName :</span></dt><dd>t2m</dd><dt><span>GRIB_dataType :</span></dt><dd>fc</dd><dt><span>GRIB_gridDefinitionDescription :</span></dt><dd>Latitude/longitude. Also called equidistant cylindrical, or Plate Carree</dd><dt><span>GRIB_gridType :</span></dt><dd>regular_ll</dd><dt><span>GRIB_iDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_iScansNegatively :</span></dt><dd>0</dd><dt><span>GRIB_jDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_jPointsAreConsecutive :</span></dt><dd>0</dd><dt><span>GRIB_jScansPositively :</span></dt><dd>0</dd><dt><span>GRIB_latitudeOfFirstGridPointInDegrees :</span></dt><dd>90.0</dd><dt><span>GRIB_latitudeOfLastGridPointInDegrees :</span></dt><dd>-90.0</dd><dt><span>GRIB_longitudeOfFirstGridPointInDegrees :</span></dt><dd>0.0</dd><dt><span>GRIB_longitudeOfLastGridPointInDegrees :</span></dt><dd>359.75</dd><dt><span>GRIB_missingValue :</span></dt><dd>3.4028234663852886e+38</dd><dt><span>GRIB_name :</span></dt><dd>2 metre temperature</dd><dt><span>GRIB_numberOfPoints :</span></dt><dd>1038240</dd><dt><span>GRIB_paramId :</span></dt><dd>167</dd><dt><span>GRIB_parameterName :</span></dt><dd>Temperature</dd><dt><span>GRIB_parameterUnits :</span></dt><dd>K</dd><dt><span>GRIB_shortName :</span></dt><dd>2t</dd><dt><span>GRIB_stepRange :</span></dt><dd>0</dd><dt><span>GRIB_stepType :</span></dt><dd>instant</dd><dt><span>GRIB_stepUnits :</span></dt><dd>1</dd><dt><span>GRIB_typeOfLevel :</span></dt><dd>heightAboveGround</dd><dt><span>GRIB_units :</span></dt><dd>K</dd><dt><span>grid_mapping :</span></dt><dd>gribfile_projection</dd><dt><span>long_name :</span></dt><dd>2 metre temperature</dd><dt><span>standard_name :</span></dt><dd>air_temperature</dd><dt><span>units :</span></dt><dd>K</dd></dl></div><div class='xr-var-data'><pre>[216992160 values with dtype=float32]</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>tcc</span></div><div class='xr-var-dims'>(longitude, latitude, valid_time)</div><div class='xr-var-dtype'>float32</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-742d1adb-17ff-4bf2-be1d-2d42909c859e' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-742d1adb-17ff-4bf2-be1d-2d42909c859e' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-732290dd-f42f-4a08-a697-62cd073d13b9' class='xr-var-data-in' type='checkbox'><label for='data-732290dd-f42f-4a08-a697-62cd073d13b9' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>GRIB_NV :</span></dt><dd>0</dd><dt><span>GRIB_Nx :</span></dt><dd>1440</dd><dt><span>GRIB_Ny :</span></dt><dd>721</dd><dt><span>GRIB_cfName :</span></dt><dd>unknown</dd><dt><span>GRIB_cfVarName :</span></dt><dd>unknown</dd><dt><span>GRIB_dataType :</span></dt><dd>fc</dd><dt><span>GRIB_gridDefinitionDescription :</span></dt><dd>Latitude/longitude. Also called equidistant cylindrical, or Plate Carree</dd><dt><span>GRIB_gridType :</span></dt><dd>regular_ll</dd><dt><span>GRIB_iDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_iScansNegatively :</span></dt><dd>0</dd><dt><span>GRIB_jDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_jPointsAreConsecutive :</span></dt><dd>0</dd><dt><span>GRIB_jScansPositively :</span></dt><dd>0</dd><dt><span>GRIB_latitudeOfFirstGridPointInDegrees :</span></dt><dd>90.0</dd><dt><span>GRIB_latitudeOfLastGridPointInDegrees :</span></dt><dd>-90.0</dd><dt><span>GRIB_longitudeOfFirstGridPointInDegrees :</span></dt><dd>0.0</dd><dt><span>GRIB_longitudeOfLastGridPointInDegrees :</span></dt><dd>359.75</dd><dt><span>GRIB_missingValue :</span></dt><dd>3.4028234663852886e+38</dd><dt><span>GRIB_name :</span></dt><dd>Total Cloud Cover</dd><dt><span>GRIB_numberOfPoints :</span></dt><dd>1038240</dd><dt><span>GRIB_paramId :</span></dt><dd>228164</dd><dt><span>GRIB_parameterName :</span></dt><dd>Total cloud cover</dd><dt><span>GRIB_parameterUnits :</span></dt><dd>%</dd><dt><span>GRIB_shortName :</span></dt><dd>tcc</dd><dt><span>GRIB_stepRange :</span></dt><dd>0</dd><dt><span>GRIB_stepType :</span></dt><dd>instant</dd><dt><span>GRIB_stepUnits :</span></dt><dd>1</dd><dt><span>GRIB_typeOfLevel :</span></dt><dd>atmosphere</dd><dt><span>GRIB_units :</span></dt><dd>%</dd><dt><span>grid_mapping :</span></dt><dd>gribfile_projection</dd><dt><span>long_name :</span></dt><dd>Total Cloud Cover</dd><dt><span>standard_name :</span></dt><dd>unknown</dd><dt><span>units :</span></dt><dd>%</dd></dl></div><div class='xr-var-data'><pre>[216992160 values with dtype=float32]</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>prate</span></div><div class='xr-var-dims'>(longitude, latitude, valid_time)</div><div class='xr-var-dtype'>float32</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-41f8370b-1744-4f75-9fc8-10c738566eb7' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-41f8370b-1744-4f75-9fc8-10c738566eb7' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-2c4ae039-5020-41b5-ba92-e8a034ce629f' class='xr-var-data-in' type='checkbox'><label for='data-2c4ae039-5020-41b5-ba92-e8a034ce629f' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>GRIB_NV :</span></dt><dd>0</dd><dt><span>GRIB_Nx :</span></dt><dd>1440</dd><dt><span>GRIB_Ny :</span></dt><dd>721</dd><dt><span>GRIB_cfName :</span></dt><dd>unknown</dd><dt><span>GRIB_cfVarName :</span></dt><dd>prate</dd><dt><span>GRIB_dataType :</span></dt><dd>fc</dd><dt><span>GRIB_gridDefinitionDescription :</span></dt><dd>Latitude/longitude. Also called equidistant cylindrical, or Plate Carree</dd><dt><span>GRIB_gridType :</span></dt><dd>regular_ll</dd><dt><span>GRIB_iDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_iScansNegatively :</span></dt><dd>0</dd><dt><span>GRIB_jDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_jPointsAreConsecutive :</span></dt><dd>0</dd><dt><span>GRIB_jScansPositively :</span></dt><dd>0</dd><dt><span>GRIB_latitudeOfFirstGridPointInDegrees :</span></dt><dd>90.0</dd><dt><span>GRIB_latitudeOfLastGridPointInDegrees :</span></dt><dd>-90.0</dd><dt><span>GRIB_longitudeOfFirstGridPointInDegrees :</span></dt><dd>0.0</dd><dt><span>GRIB_longitudeOfLastGridPointInDegrees :</span></dt><dd>359.75</dd><dt><span>GRIB_missingValue :</span></dt><dd>3.4028234663852886e+38</dd><dt><span>GRIB_name :</span></dt><dd>Precipitation rate</dd><dt><span>GRIB_numberOfPoints :</span></dt><dd>1038240</dd><dt><span>GRIB_paramId :</span></dt><dd>3059</dd><dt><span>GRIB_parameterName :</span></dt><dd>Precipitation rate</dd><dt><span>GRIB_parameterUnits :</span></dt><dd>kg m-2 s-1</dd><dt><span>GRIB_shortName :</span></dt><dd>prate</dd><dt><span>GRIB_stepRange :</span></dt><dd>0</dd><dt><span>GRIB_stepType :</span></dt><dd>instant</dd><dt><span>GRIB_stepUnits :</span></dt><dd>1</dd><dt><span>GRIB_typeOfLevel :</span></dt><dd>surface</dd><dt><span>GRIB_units :</span></dt><dd>kg m**-2 s**-1</dd><dt><span>grid_mapping :</span></dt><dd>gribfile_projection</dd><dt><span>long_name :</span></dt><dd>Precipitation rate</dd><dt><span>standard_name :</span></dt><dd>unknown</dd><dt><span>units :</span></dt><dd>kg m**-2 s**-1</dd></dl></div><div class='xr-var-data'><pre>[216992160 values with dtype=float32]</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>r2</span></div><div class='xr-var-dims'>(longitude, latitude, valid_time)</div><div class='xr-var-dtype'>float32</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-afebe971-4f83-40d8-bff7-6bc0d40926ef' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-afebe971-4f83-40d8-bff7-6bc0d40926ef' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-419f892f-1430-4605-88e4-3711e99dc92b' class='xr-var-data-in' type='checkbox'><label for='data-419f892f-1430-4605-88e4-3711e99dc92b' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>GRIB_NV :</span></dt><dd>0</dd><dt><span>GRIB_Nx :</span></dt><dd>1440</dd><dt><span>GRIB_Ny :</span></dt><dd>721</dd><dt><span>GRIB_cfName :</span></dt><dd>relative_humidity</dd><dt><span>GRIB_cfVarName :</span></dt><dd>r2</dd><dt><span>GRIB_dataType :</span></dt><dd>fc</dd><dt><span>GRIB_gridDefinitionDescription :</span></dt><dd>Latitude/longitude. Also called equidistant cylindrical, or Plate Carree</dd><dt><span>GRIB_gridType :</span></dt><dd>regular_ll</dd><dt><span>GRIB_iDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_iScansNegatively :</span></dt><dd>0</dd><dt><span>GRIB_jDirectionIncrementInDegrees :</span></dt><dd>0.25</dd><dt><span>GRIB_jPointsAreConsecutive :</span></dt><dd>0</dd><dt><span>GRIB_jScansPositively :</span></dt><dd>0</dd><dt><span>GRIB_latitudeOfFirstGridPointInDegrees :</span></dt><dd>90.0</dd><dt><span>GRIB_latitudeOfLastGridPointInDegrees :</span></dt><dd>-90.0</dd><dt><span>GRIB_longitudeOfFirstGridPointInDegrees :</span></dt><dd>0.0</dd><dt><span>GRIB_longitudeOfLastGridPointInDegrees :</span></dt><dd>359.75</dd><dt><span>GRIB_missingValue :</span></dt><dd>3.4028234663852886e+38</dd><dt><span>GRIB_name :</span></dt><dd>2 metre relative humidity</dd><dt><span>GRIB_numberOfPoints :</span></dt><dd>1038240</dd><dt><span>GRIB_paramId :</span></dt><dd>260242</dd><dt><span>GRIB_parameterName :</span></dt><dd>Relative humidity</dd><dt><span>GRIB_parameterUnits :</span></dt><dd>%</dd><dt><span>GRIB_shortName :</span></dt><dd>2r</dd><dt><span>GRIB_stepRange :</span></dt><dd>0</dd><dt><span>GRIB_stepType :</span></dt><dd>instant</dd><dt><span>GRIB_stepUnits :</span></dt><dd>1</dd><dt><span>GRIB_typeOfLevel :</span></dt><dd>heightAboveGround</dd><dt><span>GRIB_units :</span></dt><dd>%</dd><dt><span>grid_mapping :</span></dt><dd>gribfile_projection</dd><dt><span>long_name :</span></dt><dd>2 metre relative humidity</dd><dt><span>standard_name :</span></dt><dd>relative_humidity</dd><dt><span>units :</span></dt><dd>%</dd></dl></div><div class='xr-var-data'><pre>[216992160 values with dtype=float32]</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-8f79104b-3b9f-4b91-bafa-3ac28ed94db2' class='xr-section-summary-in' type='checkbox' ><label for='section-8f79104b-3b9f-4b91-bafa-3ac28ed94db2' class='xr-section-summary' >Indexes: <span>(3)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-index-name'><div>latitude</div></div><div class='xr-index-preview'>PandasIndex</div><input type='checkbox' disabled/><label></label><input id='index-57feb69d-ba5a-4fdb-aefc-76d694ff66b2' class='xr-index-data-in' type='checkbox'/><label for='index-57feb69d-ba5a-4fdb-aefc-76d694ff66b2' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(Index([ 90.0, 89.75, 89.5, 89.25, 89.0, 88.75, 88.5, 88.25, 88.0,\n",
" 87.75,\n",
" ...\n",
" -87.75, -88.0, -88.25, -88.5, -88.75, -89.0, -89.25, -89.5, -89.75,\n",
" -90.0],\n",
" dtype=&#x27;float64&#x27;, name=&#x27;latitude&#x27;, length=721))</pre></div></li><li class='xr-var-item'><div class='xr-index-name'><div>valid_time</div></div><div class='xr-index-preview'>PandasIndex</div><input type='checkbox' disabled/><label></label><input id='index-60ad28b8-fc4a-43ba-87d6-9e1a92302afb' class='xr-index-data-in' type='checkbox'/><label for='index-60ad28b8-fc4a-43ba-87d6-9e1a92302afb' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(DatetimeIndex([&#x27;2024-05-13 00:00:00&#x27;, &#x27;2024-05-13 01:00:00&#x27;,\n",
" &#x27;2024-05-13 02:00:00&#x27;, &#x27;2024-05-13 03:00:00&#x27;,\n",
" &#x27;2024-05-13 04:00:00&#x27;, &#x27;2024-05-13 05:00:00&#x27;,\n",
" &#x27;2024-05-13 06:00:00&#x27;, &#x27;2024-05-13 07:00:00&#x27;,\n",
" &#x27;2024-05-13 08:00:00&#x27;, &#x27;2024-05-13 09:00:00&#x27;,\n",
" ...\n",
" &#x27;2024-05-27 21:00:00&#x27;, &#x27;2024-05-28 00:00:00&#x27;,\n",
" &#x27;2024-05-28 03:00:00&#x27;, &#x27;2024-05-28 06:00:00&#x27;,\n",
" &#x27;2024-05-28 09:00:00&#x27;, &#x27;2024-05-28 12:00:00&#x27;,\n",
" &#x27;2024-05-28 15:00:00&#x27;, &#x27;2024-05-28 18:00:00&#x27;,\n",
" &#x27;2024-05-28 21:00:00&#x27;, &#x27;2024-05-29 00:00:00&#x27;],\n",
" dtype=&#x27;datetime64[ns]&#x27;, name=&#x27;valid_time&#x27;, length=209, freq=None))</pre></div></li><li class='xr-var-item'><div class='xr-index-name'><div>longitude</div></div><div class='xr-index-preview'>PandasIndex</div><input type='checkbox' disabled/><label></label><input id='index-68d50da9-664b-40af-a313-78d9d1e28987' class='xr-index-data-in' type='checkbox'/><label for='index-68d50da9-664b-40af-a313-78d9d1e28987' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(Index([ 0.0, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25,\n",
" ...\n",
" -2.5, -2.25, -2.0, -1.75, -1.5, -1.25, -1.0, -0.75, -0.5, -0.25],\n",
" dtype=&#x27;float64&#x27;, name=&#x27;longitude&#x27;, length=1440))</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-139290d4-1627-48fc-81f8-33f7131b94af' class='xr-section-summary-in' type='checkbox' checked><label for='section-139290d4-1627-48fc-81f8-33f7131b94af' class='xr-section-summary' >Attributes: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>description :</span></dt><dd>GFS data ingested for forecasting demo</dd></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset> Size: 4GB\n",
"Dimensions: (longitude: 1440, latitude: 721, valid_time: 209)\n",
"Coordinates:\n",
" * latitude (latitude) float64 6kB 90.0 89.75 89.5 ... -89.5 -89.75 -90.0\n",
" * valid_time (valid_time) datetime64[ns] 2kB 2024-05-13 ... 2024-05-29\n",
" time datetime64[ns] 8B ...\n",
" * longitude (longitude) float64 12kB 0.0 0.25 0.5 0.75 ... -0.75 -0.5 -0.25\n",
"Data variables:\n",
" gust (longitude, latitude, valid_time) float32 868MB ...\n",
" t2m (longitude, latitude, valid_time) float32 868MB ...\n",
" tcc (longitude, latitude, valid_time) float32 868MB ...\n",
" prate (longitude, latitude, valid_time) float32 868MB ...\n",
" r2 (longitude, latitude, valid_time) float32 868MB ...\n",
"Attributes:\n",
" description: GFS data ingested for forecasting demo"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"storage = ic.s3_storage(\n",
" bucket=\"earthmover-sample-data\",\n",
" prefix=\"icechunk/gfs/solar/2024-05-13T00:00:00-pcodec\",\n",
" from_env=True\n",
")\n",
"repo = ic.Repository.open(storage)\n",
"session = repo.readonly_session(\"main\")\n",
"store = session.store\n",
"ds_ic = xr.open_dataset(session.store, engine=xr.backends.ZarrBackendEntrypoint, zarr_format=3, consolidated=False, chunks=None);\n",
"ds_ic"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "a8042ebc-5677-4001-a28f-f39431c596e8",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"┌─────────────────────┬───────────┬───────────┬───────┬───────────┬───────┬─────────────────────┬─────────────────┐\n",
"│ valid_time │ gust │ prate │ r2 │ t2m │ tcc │ time │ geometry │\n",
"│ timestamp_ns │ float │ float │ float │ float │ float │ timestamp_ns │ geometry │\n",
"├─────────────────────┼───────────┼───────────┼───────┼───────────┼───────┼─────────────────────┼─────────────────┤\n",
"│ 2024-05-12 18:00:00 │ 6.2033625 │ 1.04e-05 │ 86.0 │ 269.8426 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-12 19:00:00 │ 6.300296 │ 7.2e-06 │ 86.3 │ 269.8121 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-12 20:00:00 │ 6.4174957 │ 4e-06 │ 83.8 │ 269.86224 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-12 21:00:00 │ 6.4212985 │ 4e-06 │ 81.2 │ 270.0631 │ 98.0 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-12 22:00:00 │ 7.029407 │ 1.6e-06 │ 84.3 │ 269.76135 │ 49.0 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-12 23:00:00 │ 7.701468 │ 0.0 │ 90.6 │ 268.10004 │ 5.0 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-13 00:00:00 │ 7.325299 │ 0.0 │ 92.8 │ 266.83868 │ 5.8 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-13 01:00:00 │ 7.016412 │ 0.0 │ 94.2 │ 266.1 │ 6.5 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-13 02:00:00 │ 6.9108753 │ 0.0 │ 94.6 │ 265.53864 │ 6.4 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ 2024-05-13 03:00:00 │ 6.7239223 │ 0.0 │ 94.3 │ 265.25797 │ 61.9 │ 2024-05-12 18:00:00 │ POINT (0 90) │\n",
"│ · │ · │ · │ · │ · │ · │ · │ · │\n",
"│ · │ · │ · │ · │ · │ · │ · │ · │\n",
"│ · │ · │ · │ · │ · │ · │ · │ · │\n",
"│ 2024-05-23 15:00:00 │ 1.6156534 │ 0.0 │ 72.0 │ 274.242 │ 95.3 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-23 18:00:00 │ 1.810498 │ 0.0 │ 70.9 │ 274.34473 │ 81.7 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-23 21:00:00 │ 1.817041 │ 0.0 │ 75.2 │ 274.463 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-24 00:00:00 │ 3.6031995 │ 0.0 │ 80.3 │ 274.15198 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-24 03:00:00 │ 4.6 │ 4.16e-05 │ 94.3 │ 273.60516 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-24 06:00:00 │ 6.7025695 │ 5.6e-06 │ 98.5 │ 274.14844 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-24 09:00:00 │ 9.019824 │ 0.000108 │ 98.7 │ 274.59375 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-24 12:00:00 │ 9.310799 │ 0.0001112 │ 98.9 │ 274.8121 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-24 15:00:00 │ 6.5135336 │ 6.4e-06 │ 98.9 │ 274.46054 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"│ 2024-05-24 18:00:00 │ 7.72677 │ 2.56e-05 │ 98.8 │ 274.40518 │ 100.0 │ 2024-05-12 18:00:00 │ POINT (0 78.25) │\n",
"├─────────────────────┴───────────┴───────────┴───────┴───────────┴───────┴─────────────────────┴─────────────────┤\n",
"│ ? rows (>9999 rows, 20 shown) 8 columns │\n",
"└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pq_fname = \"s3://earthmover-sample-data/parquet/gfs/solar/2024-05-12T18:00:00.geoparquet\"\n",
"rel = duckdb.read_parquet(pq_fname) \n",
"rel"
]
},
{
"cell_type": "markdown",
"id": "b2a2f8b0-4e8d-448d-9ad6-85ff9a6e4cef",
"metadata": {},
"source": [
"## Temporal Query"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "318c6212-9289-4d8c-af1c-95af5c834606",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ac9d3422d7f44c56a771d0838a306115",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 11.8 s, sys: 78.4 ms, total: 11.9 s\n",
"Wall time: 2.61 s\n"
]
}
],
"source": [
"%%time\n",
"df = rel.query(\"forecast_data\",\n",
" \"\"\"\n",
" SELECT valid_time, t2m\n",
" FROM forecast_data\n",
" WHERE geometry=ST_Point(-105, 40)\n",
" \"\"\"\n",
").df()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "061f9e8f-dd4f-4c7d-a683-15d1c12586f6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 62.5 ms, sys: 39.3 ms, total: 102 ms\n",
"Wall time: 172 ms\n"
]
}
],
"source": [
"%time ds_ic.t2m.isel(longitude=-105, latitude=40).compute();"
]
},
{
"cell_type": "markdown",
"id": "02b3cf15-3a59-4add-97d3-62ac6b9122d0",
"metadata": {},
"source": [
"## Spatial Query"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "cb399344-23ec-4901-82f3-28a7b1e6a5ba",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 5.71 s, sys: 644 ms, total: 6.36 s\n",
"Wall time: 1.88 s\n"
]
}
],
"source": [
"%%time\n",
"df = rel.query(\"forecast_data\",\n",
" \"\"\"\n",
" SELECT valid_time, geometry, t2m\n",
" FROM forecast_data\n",
" WHERE valid_time=make_timestamp(2024, 5, 13, 2, 0, 0)\n",
" \"\"\"\n",
").df()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "902f7416-f601-4c2e-a4fe-c5286f63d59a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 185 ms, sys: 57.6 ms, total: 243 ms\n",
"Wall time: 225 ms\n"
]
}
],
"source": [
"%time ds_ic.t2m.sel(valid_time=\"2024-05-13T02:00:00\").compute();"
]
},
{
"cell_type": "markdown",
"id": "eab5972f-1a0b-42a1-a485-5ee912087421",
"metadata": {},
"source": [
"## Temporal Mean\n",
"\n",
"For the aggregation queries, we switch to different variables to avoid biasing the results from Icechunk's aggresive caching."
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "fd299291-9d0e-4f87-a063-7ccb9976e428",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 16.3 s, sys: 405 ms, total: 16.7 s\n",
"Wall time: 2.15 s\n"
]
}
],
"source": [
"%%time\n",
"df = rel.query(\"forecast_data\",\n",
" \"\"\"\n",
" SELECT AVG(r2), geometry\n",
" FROM forecast_data\n",
" GROUP BY geometry\n",
" \"\"\"\n",
").df()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "dba927f7-3a5c-4dd1-88ac-daf46f70e1bc",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 2.95 s, sys: 447 ms, total: 3.4 s\n",
"Wall time: 1.29 s\n"
]
}
],
"source": [
"%time ds_ic.r2.mean((\"valid_time\")).compute();"
]
},
{
"cell_type": "markdown",
"id": "acefd50e-3e0c-4a67-a71e-f830a2642217",
"metadata": {},
"source": [
"## Spatial Mean"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "0187a4c6-980a-4d82-9535-4741a4ef4434",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 3.55 s, sys: 178 ms, total: 3.73 s\n",
"Wall time: 1.24 s\n"
]
}
],
"source": [
"%%time\n",
"df = rel.query(\"forecast_data\",\n",
" \"\"\"\n",
" SELECT AVG(tcc), valid_time\n",
" FROM forecast_data\n",
" GROUP BY valid_time\n",
" \"\"\"\n",
").df()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "8349f164-67f6-47b0-9dd9-02725edb785a",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 5.42 s, sys: 442 ms, total: 5.86 s\n",
"Wall time: 1.14 s\n"
]
}
],
"source": [
"%time ds_ic.tcc.mean((\"longitude\", \"latitude\")).compute();"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "93612cfe-9f93-4e81-aec3-efe23b8c67a1",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.4"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
@fr1ll
Copy link

fr1ll commented Apr 4, 2025

I’d be curious to see this against lance in place of Parquet! Maybe I’ll get time to try it myself.

@simonprovost
Copy link

It'd be nice @fr1ll to have this!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment