Skip to content

Instantly share code, notes, and snippets.

@sharkinsspatial

sharkinsspatial/variable_chunking_compressed.ipynb

Last active December 13, 2023 14:45
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save sharkinsspatial/d8423118af1157333286f23e3905f33c to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save sharkinsspatial/d8423118af1157333286f23e3905f33c to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
variable_chunking_compressed.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "0162c63e-6aa7-476c-8393-d4ff11f32c2d",
"metadata": {},
"outputs": [],
"source": [
"import xarray as xr\n",
"import numpy as np\n",
"import os\n",
"import ujson\n",
"import fsspec\n",
"from kerchunk.hdf import SingleHdf5ToZarr\n",
"from kerchunk.combine import MultiZarrToZarr\n",
"import kerchunk\n",
"import json"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "719c6ad5-a7ee-490b-9c7a-e84b76a8f50c",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt;\n",
"Dimensions: (delta_time: 17020)\n",
"Coordinates:\n",
" * delta_time (delta_time) int64 22001 22002 22003 22004 ... 39018 39019 39020\n",
"Data variables:\n",
" wat (delta_time) int64 22001 22002 22003 22004 ... 39018 39019 39020</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-6284bf11-d8f5-4667-b0be-28a2368cdf61' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-6284bf11-d8f5-4667-b0be-28a2368cdf61' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span class='xr-has-index'>delta_time</span>: 17020</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-38d504af-dd79-439a-a535-8b3062e1024a' class='xr-section-summary-in' type='checkbox' checked><label for='section-38d504af-dd79-439a-a535-8b3062e1024a' class='xr-section-summary' >Coordinates: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>delta_time</span></div><div class='xr-var-dims'>(delta_time)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>22001 22002 22003 ... 39019 39020</div><input id='attrs-5181f7f9-363f-453e-a2fe-e14bfd4acffd' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-5181f7f9-363f-453e-a2fe-e14bfd4acffd' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-e1ffd117-6d0a-4516-b96c-b1561852876e' class='xr-var-data-in' type='checkbox'><label for='data-e1ffd117-6d0a-4516-b96c-b1561852876e' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([22001, 22002, 22003, ..., 39018, 39019, 39020])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-346cc24c-824c-4aa5-b76b-d1bdb5fd5173' class='xr-section-summary-in' type='checkbox' checked><label for='section-346cc24c-824c-4aa5-b76b-d1bdb5fd5173' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>wat</span></div><div class='xr-var-dims'>(delta_time)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>22001 22002 22003 ... 39019 39020</div><input id='attrs-feebc78f-5dd2-4359-84b4-f95f6bfd3b69' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-feebc78f-5dd2-4359-84b4-f95f6bfd3b69' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-f1610488-8e65-40d0-9c90-8b4e0334687d' class='xr-var-data-in' type='checkbox'><label for='data-f1610488-8e65-40d0-9c90-8b4e0334687d' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([22001, 22002, 22003, ..., 39018, 39019, 39020])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-f6bc2fec-4ea1-4947-93d5-9ee02e240651' class='xr-section-summary-in' type='checkbox' ><label for='section-f6bc2fec-4ea1-4947-93d5-9ee02e240651' class='xr-section-summary' >Indexes: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-index-name'><div>delta_time</div></div><div class='xr-index-preview'>PandasIndex</div><div></div><input id='index-bdc0d051-23f6-44fa-969b-27f4b64b6e41' class='xr-index-data-in' type='checkbox'/><label for='index-bdc0d051-23f6-44fa-969b-27f4b64b6e41' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(Int64Index([22001, 22002, 22003, 22004, 22005, 22006, 22007, 22008, 22009,\n",
" 22010,\n",
" ...\n",
" 39011, 39012, 39013, 39014, 39015, 39016, 39017, 39018, 39019,\n",
" 39020],\n",
" dtype=&#x27;int64&#x27;, name=&#x27;delta_time&#x27;, length=17020))</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-606ba658-1aea-42d4-9274-2e6c288cf392' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-606ba658-1aea-42d4-9274-2e6c288cf392' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (delta_time: 17020)\n",
"Coordinates:\n",
" * delta_time (delta_time) int64 22001 22002 22003 22004 ... 39018 39019 39020\n",
"Data variables:\n",
" wat (delta_time) int64 22001 22002 22003 22004 ... 39018 39019 39020"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"size = 22000\n",
"time_first = xr.Dataset(\n",
" {\n",
" 'delta_time': np.arange(size),\n",
" 'wat': ('delta_time', np.arange(size)),\n",
" }\n",
")\n",
"second_size = 17020\n",
"start = size + 1\n",
"end = size + second_size + 1\n",
"time_second = xr.Dataset(\n",
" {\n",
" 'delta_time': np.arange(start, end),\n",
" 'wat': ('delta_time', np.arange(start, end)),\n",
" }\n",
")\n",
"time_second"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "06f6abe5-f187-4ac9-b7cf-846401b3c006",
"metadata": {},
"outputs": [],
"source": [
"def write_netcdfs(compressed: bool):\n",
" chunksize = 10000\n",
" encoding = {}\n",
" for variable in [\"delta_time\", \"wat\"]:\n",
" if compressed:\n",
" encoding[variable] = {\"chunksizes\": (chunksize,), \"zlib\": True}\n",
" else:\n",
" encoding[variable] = {\"chunksizes\": (chunksize,)}\n",
" if compressed:\n",
" first_name = \"first_compressed.nc\"\n",
" second_name = \"second_compressed.nc\"\n",
" else:\n",
" first_name = \"first.nc\"\n",
" second_name = \"second.nc\"\n",
" time_first.to_netcdf(first_name, encoding=encoding, engine=\"h5netcdf\", unlimited_dims=[\"delta_time\"])\n",
" time_second.to_netcdf(second_name, encoding=encoding, engine=\"h5netcdf\", unlimited_dims=[\"delta_time\"])\n",
" return [first_name, second_name]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "024603c2-eddd-419c-a6e4-4e796fa43b39",
"metadata": {},
"outputs": [],
"source": [
"def write_references(netcdfs: list[str]):\n",
" references = []\n",
" for netcdf in netcdfs:\n",
" h5chunks = SingleHdf5ToZarr(netcdf, inline_threshold=0)\n",
" reference = f\"{os.path.splitext(netcdf)[0]}.json\"\n",
" with open(reference, 'wb') as f:\n",
" f.write(ujson.dumps(h5chunks.translate()).encode())\n",
" references.append(reference)\n",
" return references"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "3718cf7f-80ad-4e2d-baac-e3bad2df50b9",
"metadata": {},
"outputs": [],
"source": [
"def write_combined_reference(compressed: bool, references: list[str]):\n",
" mzz = MultiZarrToZarr(references, concat_dims=[\"delta_time\"], inline_threshold=0)\n",
" d = mzz.translate()\n",
" if compressed:\n",
" name = \"combined_compressed.json\"\n",
" else:\n",
" name = \"combined.json\"\n",
" with open(name, 'wb') as f:\n",
" with open(references[0]) as first:\n",
" with open(references[1]) as second:\n",
" first_json = json.load(first)\n",
" second_json = json.load(second)\n",
" zarray = json.loads(d[\"refs\"][\"wat/.zarray\"])\n",
" zarray[\"chunks\"] = [[10000, 10000, 2000, 10000, 7020]]\n",
" d[\"refs\"][\"wat/.zarray\"] = json.dumps(zarray)\n",
" \n",
" d[\"refs\"][\"wat/3\"] = second_json[\"refs\"][\"wat/0\"]\n",
" if compressed:\n",
" d[\"refs\"][\"wat/2\"] = first_json[\"refs\"][\"wat/2\"]\n",
" d[\"refs\"][\"wat/4\"] = second_json[\"refs\"][\"wat/1\"]\n",
" #d[\"refs\"][\"wat/2\"] = [first_json[\"refs\"][\"wat/2\"][0], first_json[\"refs\"][\"wat/2\"][1], 3092]\n",
" #d[\"refs\"][\"wat/4\"] = [second_json[\"refs\"][\"wat/1\"][0], second_json[\"refs\"][\"wat/1\"][1], 25839]\n",
" else:\n",
" d[\"refs\"][\"wat/2\"] = [first_json[\"refs\"][\"wat/2\"][0], first_json[\"refs\"][\"wat/2\"][1], 16000]\n",
" d[\"refs\"][\"wat/4\"] = [second_json[\"refs\"][\"wat/1\"][0], second_json[\"refs\"][\"wat/1\"][1], 56160]\n",
" f.write(ujson.dumps(d).encode())\n",
" return name"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "f24ef0c4-9e55-44e9-ae2f-c3db65eacb53",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['first_compressed.json', 'second_compressed.json']\n"
]
},
{
"data": {
"text/plain": [
"'combined_compressed.json'"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"netcdfs = write_netcdfs(compressed=True)\n",
"references = write_references(netcdfs)\n",
"print(references)\n",
"combined_reference_compressed = write_combined_reference(compressed=True, references=references)\n",
"combined_reference_compressed"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "2accc76b-79d8-4cca-bb6d-9f1f10186ee5",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt;\n",
"Dimensions: (delta_time: 39020)\n",
"Coordinates:\n",
" * delta_time (delta_time) int64 0 1 2 3 4 5 ... 39016 39017 39018 39019 39020\n",
"Data variables:\n",
" wat (delta_time) int64 ...</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-bd9d4b2e-811b-4445-b1a2-fe1597a293b8' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-bd9d4b2e-811b-4445-b1a2-fe1597a293b8' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span class='xr-has-index'>delta_time</span>: 39020</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-7c9b54be-eafd-452f-9775-afa4aa865770' class='xr-section-summary-in' type='checkbox' checked><label for='section-7c9b54be-eafd-452f-9775-afa4aa865770' class='xr-section-summary' >Coordinates: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>delta_time</span></div><div class='xr-var-dims'>(delta_time)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 ... 39017 39018 39019 39020</div><input id='attrs-86f0107d-8e3c-4ff5-9c78-9bb22cbe3d64' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-86f0107d-8e3c-4ff5-9c78-9bb22cbe3d64' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-4ae71c62-0382-4896-a9cd-08d8092d260f' class='xr-var-data-in' type='checkbox'><label for='data-4ae71c62-0382-4896-a9cd-08d8092d260f' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, ..., 39018, 39019, 39020])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-6283d1da-b39d-4d00-8806-a111b6276d5e' class='xr-section-summary-in' type='checkbox' checked><label for='section-6283d1da-b39d-4d00-8806-a111b6276d5e' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>wat</span></div><div class='xr-var-dims'>(delta_time)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-c37a152f-1c74-4799-a818-3a660b4d9914' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-c37a152f-1c74-4799-a818-3a660b4d9914' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-fb43ef1d-3a3f-454f-95d1-cee1905bbf39' class='xr-var-data-in' type='checkbox'><label for='data-fb43ef1d-3a3f-454f-95d1-cee1905bbf39' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>[39020 values with dtype=int64]</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-a537ac60-78f1-4068-b753-eae626f6d362' class='xr-section-summary-in' type='checkbox' ><label for='section-a537ac60-78f1-4068-b753-eae626f6d362' class='xr-section-summary' >Indexes: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-index-name'><div>delta_time</div></div><div class='xr-index-preview'>PandasIndex</div><div></div><input id='index-296d7f91-15dc-4d1b-afbd-9d8b4fea6972' class='xr-index-data-in' type='checkbox'/><label for='index-296d7f91-15dc-4d1b-afbd-9d8b4fea6972' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(Int64Index([ 0, 1, 2, 3, 4, 5, 6, 7, 8,\n",
" 9,\n",
" ...\n",
" 39011, 39012, 39013, 39014, 39015, 39016, 39017, 39018, 39019,\n",
" 39020],\n",
" dtype=&#x27;int64&#x27;, name=&#x27;delta_time&#x27;, length=39020))</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-eb92c070-ebf0-4bd8-8743-38c463c2abcc' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-eb92c070-ebf0-4bd8-8743-38c463c2abcc' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (delta_time: 39020)\n",
"Coordinates:\n",
" * delta_time (delta_time) int64 0 1 2 3 4 5 ... 39016 39017 39018 39019 39020\n",
"Data variables:\n",
" wat (delta_time) int64 ..."
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fs = fsspec.filesystem(\"reference\", fo=\"combined_compressed.json\")\n",
"m = fs.get_mapper(\"\")\n",
"ds = xr.open_dataset(m, engine='zarr', backend_kwargs={'consolidated':False})\n",
"ds"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "db21c4e3-4116-4160-abd3-c018be097995",
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "cannot reshape array of size 10000 into shape (2000,)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m np\u001b[38;5;241m.\u001b[39mcount_nonzero(\u001b[43mds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwat\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m)\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/core/dataarray.py:771\u001b[0m, in \u001b[0;36mDataArray.values\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 762\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 763\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mvalues\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m np\u001b[38;5;241m.\u001b[39mndarray:\n\u001b[1;32m 764\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 765\u001b[0m \u001b[38;5;124;03m The array's data as a numpy.ndarray.\u001b[39;00m\n\u001b[1;32m 766\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 769\u001b[0m \u001b[38;5;124;03m type does not support coercion like this (e.g. cupy).\u001b[39;00m\n\u001b[1;32m 770\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 771\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/core/variable.py:556\u001b[0m, in \u001b[0;36mVariable.values\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 554\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mvalues\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 555\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"The variable's data as a numpy.ndarray\"\"\"\u001b[39;00m\n\u001b[0;32m--> 556\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_as_array_or_item\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_data\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/core/variable.py:306\u001b[0m, in \u001b[0;36m_as_array_or_item\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m 292\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_as_array_or_item\u001b[39m(data):\n\u001b[1;32m 293\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Return the given values as a numpy array, or as an individual item if\u001b[39;00m\n\u001b[1;32m 294\u001b[0m \u001b[38;5;124;03m it's a 0d datetime64 or timedelta64 array.\u001b[39;00m\n\u001b[1;32m 295\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[38;5;124;03m TODO: remove this (replace with np.asarray) once these issues are fixed\u001b[39;00m\n\u001b[1;32m 305\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 306\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43masarray\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 307\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m data\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 308\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m data\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39mkind \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mM\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/core/indexing.py:696\u001b[0m, in \u001b[0;36mMemoryCachedArray.__array__\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m 695\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__array__\u001b[39m(\u001b[38;5;28mself\u001b[39m, dtype: np\u001b[38;5;241m.\u001b[39mtyping\u001b[38;5;241m.\u001b[39mDTypeLike \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m np\u001b[38;5;241m.\u001b[39mndarray:\n\u001b[0;32m--> 696\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, dtype\u001b[38;5;241m=\u001b[39mdtype)\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/core/indexing.py:699\u001b[0m, in \u001b[0;36mMemoryCachedArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 698\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 699\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ensure_cached\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 700\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39marray\u001b[38;5;241m.\u001b[39mget_duck_array()\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/core/indexing.py:693\u001b[0m, in \u001b[0;36mMemoryCachedArray._ensure_cached\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 692\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_ensure_cached\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 693\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39marray \u001b[38;5;241m=\u001b[39m as_indexable(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/core/indexing.py:667\u001b[0m, in \u001b[0;36mCopyOnWriteArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 666\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 667\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/core/indexing.py:554\u001b[0m, in \u001b[0;36mLazilyIndexedArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 554\u001b[0m array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 555\u001b[0m \u001b[38;5;66;03m# self.array[self.key] is now a numpy array when\u001b[39;00m\n\u001b[1;32m 556\u001b[0m \u001b[38;5;66;03m# self.array is a BackendArray subclass\u001b[39;00m\n\u001b[1;32m 557\u001b[0m \u001b[38;5;66;03m# and self.key is BasicIndexer((slice(None, None, None),))\u001b[39;00m\n\u001b[1;32m 558\u001b[0m \u001b[38;5;66;03m# so we need the explicit check for ExplicitlyIndexed\u001b[39;00m\n\u001b[1;32m 559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(array, ExplicitlyIndexed):\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/xarray/backends/zarr.py:94\u001b[0m, in \u001b[0;36mZarrArrayWrapper.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 92\u001b[0m array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_array()\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(key, indexing\u001b[38;5;241m.\u001b[39mBasicIndexer):\n\u001b[0;32m---> 94\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43marray\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtuple\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 95\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(key, indexing\u001b[38;5;241m.\u001b[39mVectorizedIndexer):\n\u001b[1;32m 96\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m array\u001b[38;5;241m.\u001b[39mvindex[\n\u001b[1;32m 97\u001b[0m indexing\u001b[38;5;241m.\u001b[39m_arrayize_vectorized_indexer(key, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape)\u001b[38;5;241m.\u001b[39mtuple\n\u001b[1;32m 98\u001b[0m ]\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/zarr/core.py:844\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 842\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_orthogonal_selection(pure_selection, fields\u001b[38;5;241m=\u001b[39mfields)\n\u001b[1;32m 843\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 844\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_basic_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 845\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/zarr/core.py:970\u001b[0m, in \u001b[0;36mArray.get_basic_selection\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 968\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_basic_selection_zd(selection\u001b[38;5;241m=\u001b[39mselection, out\u001b[38;5;241m=\u001b[39mout, fields\u001b[38;5;241m=\u001b[39mfields)\n\u001b[1;32m 969\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 970\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_basic_selection_nd\u001b[49m\u001b[43m(\u001b[49m\u001b[43mselection\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mselection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/zarr/core.py:1012\u001b[0m, in \u001b[0;36mArray._get_basic_selection_nd\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 1006\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_get_basic_selection_nd\u001b[39m(\u001b[38;5;28mself\u001b[39m, selection, out\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, fields\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 1007\u001b[0m \u001b[38;5;66;03m# implementation of basic selection for array with at least one dimension\u001b[39;00m\n\u001b[1;32m 1008\u001b[0m \n\u001b[1;32m 1009\u001b[0m \u001b[38;5;66;03m# setup indexer\u001b[39;00m\n\u001b[1;32m 1010\u001b[0m indexer \u001b[38;5;241m=\u001b[39m BasicIndexer(selection, \u001b[38;5;28mself\u001b[39m)\n\u001b[0;32m-> 1012\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/zarr/core.py:1388\u001b[0m, in \u001b[0;36mArray._get_selection\u001b[0;34m(self, indexer, out, fields)\u001b[0m\n\u001b[1;32m 1385\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m math\u001b[38;5;241m.\u001b[39mprod(out_shape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 1386\u001b[0m \u001b[38;5;66;03m# allow storage to get multiple items at once\u001b[39;00m\n\u001b[1;32m 1387\u001b[0m lchunk_coords, lchunk_selection, lout_selection \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mindexer)\n\u001b[0;32m-> 1388\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_chunk_getitems\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1389\u001b[0m \u001b[43m \u001b[49m\u001b[43mlchunk_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1390\u001b[0m \u001b[43m \u001b[49m\u001b[43mlchunk_selection\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1391\u001b[0m \u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1392\u001b[0m \u001b[43m \u001b[49m\u001b[43mlout_selection\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1393\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_axes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdrop_axes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1394\u001b[0m \u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1395\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1396\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m out\u001b[38;5;241m.\u001b[39mshape:\n\u001b[1;32m 1397\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/zarr/core.py:2232\u001b[0m, in \u001b[0;36mArray._chunk_getitems\u001b[0;34m(self, lchunk_coords, lchunk_selection, out, lout_selection, drop_axes, fields)\u001b[0m\n\u001b[1;32m 2228\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m ckey, coords, chunk_select, out_select \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 2229\u001b[0m ckeys, lchunk_coords, lchunk_selection, lout_selection\n\u001b[1;32m 2230\u001b[0m ):\n\u001b[1;32m 2231\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m ckey \u001b[38;5;129;01min\u001b[39;00m cdatas:\n\u001b[0;32m-> 2232\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process_chunk\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2233\u001b[0m \u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2234\u001b[0m \u001b[43m \u001b[49m\u001b[43mcdatas\u001b[49m\u001b[43m[\u001b[49m\u001b[43mckey\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2235\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunk_select\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2236\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_axes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2237\u001b[0m \u001b[43m \u001b[49m\u001b[43mout_is_ndarray\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2238\u001b[0m \u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2239\u001b[0m \u001b[43m \u001b[49m\u001b[43mout_select\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2240\u001b[0m \u001b[43m \u001b[49m\u001b[43mpartial_read_decode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpartial_read_decode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2241\u001b[0m \u001b[43m \u001b[49m\u001b[43mcoords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcoords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2242\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2243\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 2244\u001b[0m \u001b[38;5;66;03m# check exception type\u001b[39;00m\n\u001b[1;32m 2245\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fill_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/zarr/core.py:2143\u001b[0m, in \u001b[0;36mArray._process_chunk\u001b[0;34m(self, out, cdata, chunk_selection, drop_axes, out_is_ndarray, fields, out_selection, partial_read_decode, coords)\u001b[0m\n\u001b[1;32m 2141\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ArrayIndexError:\n\u001b[1;32m 2142\u001b[0m cdata \u001b[38;5;241m=\u001b[39m cdata\u001b[38;5;241m.\u001b[39mread_full()\n\u001b[0;32m-> 2143\u001b[0m chunk \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_decode_chunk\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexpected_shape\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2145\u001b[0m \u001b[38;5;66;03m# select data from chunk\u001b[39;00m\n\u001b[1;32m 2146\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m fields:\n",
"File \u001b[0;32m~/projects/icesat_kerchunk/venv/lib/python3.9/site-packages/zarr/core.py:2435\u001b[0m, in \u001b[0;36mArray._decode_chunk\u001b[0;34m(self, cdata, start, nitems, expected_shape)\u001b[0m\n\u001b[1;32m 2433\u001b[0m \u001b[38;5;66;03m# ensure correct chunk shape\u001b[39;00m\n\u001b[1;32m 2434\u001b[0m chunk \u001b[38;5;241m=\u001b[39m chunk\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, order\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mA\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 2435\u001b[0m chunk \u001b[38;5;241m=\u001b[39m \u001b[43mchunk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreshape\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexpected_shape\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_chunks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_order\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2437\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m chunk\n",
"\u001b[0;31mValueError\u001b[0m: cannot reshape array of size 10000 into shape (2000,)"
]
}
],
"source": [
"np.count_nonzero(ds.wat.values == 0)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "61db16a2-3b32-4023-bd16-4c70943f2f43",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'combined.json'"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"netcdfs = write_netcdfs(compressed=False)\n",
"references = write_references(netcdfs)\n",
"combined_reference = write_combined_reference(compressed=False, references=references)\n",
"combined_reference"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "2a537a81-35c1-4eca-8c89-6eaf37df3787",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in jupyterlab.\n",
" *\n",
" */\n",
"\n",
":root {\n",
" --xr-font-color0: var(--jp-content-font-color0, rgba(0, 0, 0, 1));\n",
" --xr-font-color2: var(--jp-content-font-color2, rgba(0, 0, 0, 0.54));\n",
" --xr-font-color3: var(--jp-content-font-color3, rgba(0, 0, 0, 0.38));\n",
" --xr-border-color: var(--jp-border-color2, #e0e0e0);\n",
" --xr-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
" --xr-background-color: var(--jp-layout-color0, white);\n",
" --xr-background-color-row-even: var(--jp-layout-color1, white);\n",
" --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n",
"}\n",
"\n",
"html[theme=dark],\n",
"body[data-theme=dark],\n",
"body.vscode-dark {\n",
" --xr-font-color0: rgba(255, 255, 255, 1);\n",
" --xr-font-color2: rgba(255, 255, 255, 0.54);\n",
" --xr-font-color3: rgba(255, 255, 255, 0.38);\n",
" --xr-border-color: #1F1F1F;\n",
" --xr-disabled-color: #515151;\n",
" --xr-background-color: #111111;\n",
" --xr-background-color-row-even: #111111;\n",
" --xr-background-color-row-odd: #313131;\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-array-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 20px 20px;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: '►';\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: '▼';\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
" padding-bottom: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: '(';\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: ')';\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: ',';\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" background-color: var(--xr-background-color) !important;\n",
" padding-bottom: 5px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.Dataset&gt;\n",
"Dimensions: (delta_time: 39020)\n",
"Coordinates:\n",
" * delta_time (delta_time) int64 0 1 2 3 4 5 ... 39016 39017 39018 39019 39020\n",
"Data variables:\n",
" wat (delta_time) int64 ...</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-8f9eb351-c79c-4d48-9d5b-fdcbd2bc8db5' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-8f9eb351-c79c-4d48-9d5b-fdcbd2bc8db5' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span class='xr-has-index'>delta_time</span>: 39020</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-03d7e6d7-71a4-4b39-9ed7-aa5df8edf719' class='xr-section-summary-in' type='checkbox' checked><label for='section-03d7e6d7-71a4-4b39-9ed7-aa5df8edf719' class='xr-section-summary' >Coordinates: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>delta_time</span></div><div class='xr-var-dims'>(delta_time)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>0 1 2 3 ... 39017 39018 39019 39020</div><input id='attrs-a03ea5f8-4684-4bbe-b97a-3897f7b0ad89' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-a03ea5f8-4684-4bbe-b97a-3897f7b0ad89' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-398fcb12-3d35-4ef1-9727-86998a5d314d' class='xr-var-data-in' type='checkbox'><label for='data-398fcb12-3d35-4ef1-9727-86998a5d314d' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>array([ 0, 1, 2, ..., 39018, 39019, 39020])</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-60d743ff-5d60-455d-b484-6c14df8f95b3' class='xr-section-summary-in' type='checkbox' checked><label for='section-60d743ff-5d60-455d-b484-6c14df8f95b3' class='xr-section-summary' >Data variables: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>wat</span></div><div class='xr-var-dims'>(delta_time)</div><div class='xr-var-dtype'>int64</div><div class='xr-var-preview xr-preview'>...</div><input id='attrs-1c6fdb7f-c478-44d9-9a10-a36c4f5dfe94' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-1c6fdb7f-c478-44d9-9a10-a36c4f5dfe94' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-b524f8bf-a8c6-4d0b-b2e6-ded7d0171951' class='xr-var-data-in' type='checkbox'><label for='data-b524f8bf-a8c6-4d0b-b2e6-ded7d0171951' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><pre>[39020 values with dtype=int64]</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-cf040f0a-6337-453b-ba80-da25b2aac306' class='xr-section-summary-in' type='checkbox' ><label for='section-cf040f0a-6337-453b-ba80-da25b2aac306' class='xr-section-summary' >Indexes: <span>(1)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-index-name'><div>delta_time</div></div><div class='xr-index-preview'>PandasIndex</div><div></div><input id='index-96716da2-6126-49fe-9d82-09d793607f98' class='xr-index-data-in' type='checkbox'/><label for='index-96716da2-6126-49fe-9d82-09d793607f98' title='Show/Hide index repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-index-data'><pre>PandasIndex(Int64Index([ 0, 1, 2, 3, 4, 5, 6, 7, 8,\n",
" 9,\n",
" ...\n",
" 39011, 39012, 39013, 39014, 39015, 39016, 39017, 39018, 39019,\n",
" 39020],\n",
" dtype=&#x27;int64&#x27;, name=&#x27;delta_time&#x27;, length=39020))</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-f29cd748-c4b9-412b-83c2-b65b536ec574' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-f29cd748-c4b9-412b-83c2-b65b536ec574' class='xr-section-summary' title='Expand/collapse section'>Attributes: <span>(0)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (delta_time: 39020)\n",
"Coordinates:\n",
" * delta_time (delta_time) int64 0 1 2 3 4 5 ... 39016 39017 39018 39019 39020\n",
"Data variables:\n",
" wat (delta_time) int64 ..."
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fs = fsspec.filesystem(\"reference\", fo=\"combined.json\")\n",
"m = fs.get_mapper(\"\")\n",
"ds = xr.open_dataset(m, engine='zarr', backend_kwargs={'consolidated':False})\n",
"ds"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "a6f16a50-1228-4917-8321-f6b6a5d1a6fd",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.count_nonzero(ds.wat.values == 0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1a29ce59-6fc3-46fe-ac52-60ec92102465",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment