Skip to content

Instantly share code, notes, and snippets.

@calebrob6

calebrob6/load_aef_with_xarray.ipynb

Created March 11, 2026 16:58
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save calebrob6/48a6f17ac813753574a130b05a44986d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save calebrob6/48a6f17ac813753574a130b05a44986d to your computer and use it in GitHub Desktop.
Download ZIP
How to load parts of the AEF embeddings via xarray
Display the source blob
Display the rendered blob
Raw
load_aef_with_xarray.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 48,
"id": "8dfba42d",
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"import warnings\n",
"import numpy as np\n",
"import xarray as xr\n",
"import zarr\n",
"import zarr.storage\n",
"from zarr.errors import ZarrUserWarning\n",
"import rasterix\n",
"from rasterix import RasterIndex\n",
"from affine import Affine\n",
"from pyproj import CRS\n",
"\n",
"from sklearn.decomposition import PCA\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# gets rid of a bunch of warnings like \"ZarrUserWarning: Object at .checkpoint.2018.json is not recognized as a component of a Zarr hierarchy.\"\n",
"warnings.filterwarnings(\"ignore\", category=ZarrUserWarning)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "e81fada0",
"metadata": {},
"outputs": [],
"source": [
"STORE_URL = \"s3://us-west-2.opendata.source.coop/tge-labs/aef-mosaic\""
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "c1613f43",
"metadata": {},
"outputs": [],
"source": [
"def open_aef_mosaic():\n",
" tic = time.time()\n",
"\n",
" store = zarr.storage.FsspecStore.from_url(\n",
" STORE_URL,\n",
" storage_options={\"anon\": True},\n",
" read_only=True,\n",
" )\n",
" ds = xr.open_zarr(store, zarr_format=3, consolidated=False)\n",
"\n",
" print(f\" Opened in {time.time() - tic:.2f}s\")\n",
" return ds\n",
"\n",
"def assign_rasterix_index(ds):\n",
" attrs = ds[\"embeddings\"].attrs\n",
" affine = Affine(*attrs[\"spatial:transform\"])\n",
" crs = CRS.from_user_input(attrs[\"proj:code\"])\n",
" height, width = attrs[\"spatial:shape\"]\n",
"\n",
" raster_idx = RasterIndex.from_transform(\n",
" affine, width=width, height=height,\n",
" x_dim=\"x\", y_dim=\"y\", crs=crs,\n",
" )\n",
"\n",
" emb = ds[\"embeddings\"].drop_vars([\"x\", \"y\"], errors=\"ignore\")\n",
" emb = emb.assign_coords(raster_idx.create_variables())\n",
" return emb"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "19aca967",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Opened in 1.40s\n"
]
}
],
"source": [
"ds = open_aef_mosaic()\n",
"embedding_ds = assign_rasterix_index(ds)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0beda606",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in notebooks */\n",
"\n",
":root {\n",
" --xr-font-color0: var(\n",
" --jp-content-font-color0,\n",
" var(--pst-color-text-base rgba(0, 0, 0, 1))\n",
" );\n",
" --xr-font-color2: var(\n",
" --jp-content-font-color2,\n",
" var(--pst-color-text-base, rgba(0, 0, 0, 0.54))\n",
" );\n",
" --xr-font-color3: var(\n",
" --jp-content-font-color3,\n",
" var(--pst-color-text-base, rgba(0, 0, 0, 0.38))\n",
" );\n",
" --xr-border-color: var(\n",
" --jp-border-color2,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 10))\n",
" );\n",
" --xr-disabled-color: var(\n",
" --jp-layout-color3,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 40))\n",
" );\n",
" --xr-background-color: var(\n",
" --jp-layout-color0,\n",
" var(--pst-color-on-background, white)\n",
" );\n",
" --xr-background-color-row-even: var(\n",
" --jp-layout-color1,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 5))\n",
" );\n",
" --xr-background-color-row-odd: var(\n",
" --jp-layout-color2,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 15))\n",
" );\n",
"}\n",
"\n",
"html[theme=\"dark\"],\n",
"html[data-theme=\"dark\"],\n",
"body[data-theme=\"dark\"],\n",
"body.vscode-dark {\n",
" --xr-font-color0: var(\n",
" --jp-content-font-color0,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 1))\n",
" );\n",
" --xr-font-color2: var(\n",
" --jp-content-font-color2,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 0.54))\n",
" );\n",
" --xr-font-color3: var(\n",
" --jp-content-font-color3,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 0.38))\n",
" );\n",
" --xr-border-color: var(\n",
" --jp-border-color2,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 10))\n",
" );\n",
" --xr-disabled-color: var(\n",
" --jp-layout-color3,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 40))\n",
" );\n",
" --xr-background-color: var(\n",
" --jp-layout-color0,\n",
" var(--pst-color-on-background, #111111)\n",
" );\n",
" --xr-background-color-row-even: var(\n",
" --jp-layout-color1,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 5))\n",
" );\n",
" --xr-background-color-row-odd: var(\n",
" --jp-layout-color2,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 15))\n",
" );\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
" line-height: 1.6;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-obj-name,\n",
".xr-group-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-group-name::before {\n",
" content: \"📁\";\n",
" padding-right: 0.3em;\n",
"}\n",
"\n",
".xr-group-name,\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 0 20px 0 20px;\n",
" margin-block-start: 0;\n",
" margin-block-end: 0;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: inline-block;\n",
" opacity: 0;\n",
" height: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
" border: 2px solid transparent !important;\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:focus + label {\n",
" border: 2px solid var(--xr-font-color0) !important;\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: \"►\";\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: \"▼\";\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-top: 4px;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-group-box {\n",
" display: inline-grid;\n",
" grid-template-columns: 0px 20px auto;\n",
" width: 100%;\n",
"}\n",
"\n",
".xr-group-box-vline {\n",
" grid-column-start: 1;\n",
" border-right: 0.2em solid;\n",
" border-color: var(--xr-border-color);\n",
" width: 0px;\n",
"}\n",
"\n",
".xr-group-box-hline {\n",
" grid-column-start: 2;\n",
" grid-row-start: 1;\n",
" height: 1em;\n",
" width: 20px;\n",
" border-bottom: 0.2em solid;\n",
" border-color: var(--xr-border-color);\n",
"}\n",
"\n",
".xr-group-box-contents {\n",
" grid-column-start: 3;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: \"(\";\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: \")\";\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: \",\";\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" border-color: var(--xr-background-color-row-odd);\n",
" margin-bottom: 0;\n",
" padding-top: 2px;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
" border-color: var(--xr-background-color-row-even);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" border-top: 2px dotted var(--xr-background-color);\n",
" padding-bottom: 20px !important;\n",
" padding-top: 10px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in + label,\n",
".xr-var-data-in + label,\n",
".xr-index-data-in + label {\n",
" padding: 0 1px;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-data > pre,\n",
".xr-index-data > pre,\n",
".xr-var-data > table > tbody > tr {\n",
" background-color: transparent !important;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked + label > .xr-icon-file-text2,\n",
".xr-var-data-in:checked + label > .xr-icon-database,\n",
".xr-index-data-in:checked + label > .xr-icon-database {\n",
" color: var(--xr-font-color0);\n",
" filter: drop-shadow(1px 1px 5px var(--xr-font-color2));\n",
" stroke-width: 0.8px;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray &#x27;embeddings&#x27; (time: 7, embedding: 64, y: 1859584, x: 4009984)&gt; Size: 3PB\n",
"[3340692134821888 values with dtype=int8]\n",
"Coordinates:\n",
" * time (time) int32 28B 2018 2019 2020 2021 2022 2023 2024\n",
" * y (y) float64 15MB 83.69 83.69 83.69 83.69 ... -83.36 -83.36 -83.36\n",
" * x (x) float64 32MB -180.0 -180.0 -180.0 -180.0 ... 180.2 180.2 180.2\n",
"Dimensions without coordinates: embedding\n",
"Attributes:\n",
" proj:code: EPSG:4326\n",
" spatial:dimensions: [&#x27;y&#x27;, &#x27;x&#x27;]\n",
" spatial:transform: [8.983111749910169e-05, 0.0, -180.0, 0.0, -8.983...\n",
" spatial:transform_type: affine\n",
" spatial:bbox: [-180.0, -83.36280346631479, 180.2213438735178, ...\n",
" spatial:shape: [1859584, 4009984]\n",
" spatial:registration: pixel\n",
" embedding_dimensions: [&#x27;A00&#x27;, &#x27;A01&#x27;, &#x27;A02&#x27;, &#x27;A03&#x27;, &#x27;A04&#x27;, &#x27;A05&#x27;, &#x27;A06&#x27;...\n",
" zarr_conventions: [{&#x27;uuid&#x27;: &#x27;f17cb550-5864-4468-aeb7-f3180cfb622f&#x27;...\n",
" _zarrs: {&#x27;description&#x27;: &#x27;This array was created with zar...</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-obj-name'>&#x27;embeddings&#x27;</div><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 7</li><li><span>embedding</span>: 64</li><li><span class='xr-has-index'>y</span>: 1859584</li><li><span class='xr-has-index'>x</span>: 4009984</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-1284aac0-e72d-4fb1-b044-bd1862ff55ca' class='xr-array-in' type='checkbox' checked><label for='section-1284aac0-e72d-4fb1-b044-bd1862ff55ca' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>...</span></div><div class='xr-array-data'><pre>[3340692134821888 values with dtype=int8]</pre></div></div></li><li class='xr-section-item'><input id='section-26a4c447-93df-4c23-9f89-00a99f7d9cc7' class='xr-section-summary-in' type='checkbox' checked><label for='section-26a4c447-93df-4c23-9f89-00a99f7d9cc7' class='xr-section-summary' >Coordinates: <span>(3)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>int32</div><div class='xr-var-preview xr-preview'>2018 2019 2020 2021 2022 2023 2024</div><input id='attrs-7b1f9d7a-f47f-46d8-84cf-9131affc38c7' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-7b1f9d7a-f47f-46d8-84cf-9131affc38c7' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-9022402e-e6f7-45c6-a373-99cc1d672a09' class='xr-var-data-in' type='checkbox'><label for='data-9022402e-e6f7-45c6-a373-99cc1d672a09' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>_zarrs :</span></dt><dd>{&#x27;description&#x27;: &#x27;This array was created with zarrs&#x27;, &#x27;repository&#x27;: &#x27;https://github.com/zarrs/zarrs&#x27;, &#x27;version&#x27;: &#x27;0.23.0&#x27;}</dd></dl></div><div class='xr-var-data'><pre>array([2018, 2019, 2020, 2021, 2022, 2023, 2024], dtype=int32)</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>y</span></div><div class='xr-var-dims'>(y)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>83.69 83.69 83.69 ... -83.36 -83.36</div><input id='attrs-0d894642-7f57-496a-ba9a-b9506b0cc6fe' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-0d894642-7f57-496a-ba9a-b9506b0cc6fe' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-23a6ea2a-7d89-4d96-b910-26c4678ee669' class='xr-var-data-in' type='checkbox'><label for='data-23a6ea2a-7d89-4d96-b910-26c4678ee669' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>latitude</dd><dt><span>long_name :</span></dt><dd>latitude coordinate</dd><dt><span>units :</span></dt><dd>degrees_north</dd><dt><span>axis :</span></dt><dd>Y</dd></dl></div><div class='xr-var-data'><pre>array([ 83.68566 , 83.685571, 83.685481, ..., -83.362579, -83.362669,\n",
" -83.362759], shape=(1859584,))</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>x</span></div><div class='xr-var-dims'>(x)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>-180.0 -180.0 ... 180.2 180.2</div><input id='attrs-fef7f351-546e-4c3a-a576-056d7faa41a2' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-fef7f351-546e-4c3a-a576-056d7faa41a2' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-40a3fa3b-286a-4196-8c17-0b92437481f1' class='xr-var-data-in' type='checkbox'><label for='data-40a3fa3b-286a-4196-8c17-0b92437481f1' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>longitude</dd><dt><span>long_name :</span></dt><dd>longitude coordinate</dd><dt><span>units :</span></dt><dd>degrees_east</dd><dt><span>axis :</span></dt><dd>X</dd></dl></div><div class='xr-var-data'><pre>array([-179.999955, -179.999865, -179.999775, ..., 180.221119, 180.221209,\n",
" 180.221299], shape=(4009984,))</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-ae087b78-f37a-4363-a03d-8d04a071dfbc' class='xr-section-summary-in' type='checkbox' ><label for='section-ae087b78-f37a-4363-a03d-8d04a071dfbc' class='xr-section-summary' >Attributes: <span>(10)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>proj:code :</span></dt><dd>EPSG:4326</dd><dt><span>spatial:dimensions :</span></dt><dd>[&#x27;y&#x27;, &#x27;x&#x27;]</dd><dt><span>spatial:transform :</span></dt><dd>[8.983111749910169e-05, 0.0, -180.0, 0.0, -8.983111749910169e-05, 83.68570533713473]</dd><dt><span>spatial:transform_type :</span></dt><dd>affine</dd><dt><span>spatial:bbox :</span></dt><dd>[-180.0, -83.36280346631479, 180.2213438735178, 83.68570533713473]</dd><dt><span>spatial:shape :</span></dt><dd>[1859584, 4009984]</dd><dt><span>spatial:registration :</span></dt><dd>pixel</dd><dt><span>embedding_dimensions :</span></dt><dd>[&#x27;A00&#x27;, &#x27;A01&#x27;, &#x27;A02&#x27;, &#x27;A03&#x27;, &#x27;A04&#x27;, &#x27;A05&#x27;, &#x27;A06&#x27;, &#x27;A07&#x27;, &#x27;A08&#x27;, &#x27;A09&#x27;, &#x27;A10&#x27;, &#x27;A11&#x27;, &#x27;A12&#x27;, &#x27;A13&#x27;, &#x27;A14&#x27;, &#x27;A15&#x27;, &#x27;A16&#x27;, &#x27;A17&#x27;, &#x27;A18&#x27;, &#x27;A19&#x27;, &#x27;A20&#x27;, &#x27;A21&#x27;, &#x27;A22&#x27;, &#x27;A23&#x27;, &#x27;A24&#x27;, &#x27;A25&#x27;, &#x27;A26&#x27;, &#x27;A27&#x27;, &#x27;A28&#x27;, &#x27;A29&#x27;, &#x27;A30&#x27;, &#x27;A31&#x27;, &#x27;A32&#x27;, &#x27;A33&#x27;, &#x27;A34&#x27;, &#x27;A35&#x27;, &#x27;A36&#x27;, &#x27;A37&#x27;, &#x27;A38&#x27;, &#x27;A39&#x27;, &#x27;A40&#x27;, &#x27;A41&#x27;, &#x27;A42&#x27;, &#x27;A43&#x27;, &#x27;A44&#x27;, &#x27;A45&#x27;, &#x27;A46&#x27;, &#x27;A47&#x27;, &#x27;A48&#x27;, &#x27;A49&#x27;, &#x27;A50&#x27;, &#x27;A51&#x27;, &#x27;A52&#x27;, &#x27;A53&#x27;, &#x27;A54&#x27;, &#x27;A55&#x27;, &#x27;A56&#x27;, &#x27;A57&#x27;, &#x27;A58&#x27;, &#x27;A59&#x27;, &#x27;A60&#x27;, &#x27;A61&#x27;, &#x27;A62&#x27;, &#x27;A63&#x27;]</dd><dt><span>zarr_conventions :</span></dt><dd>[{&#x27;uuid&#x27;: &#x27;f17cb550-5864-4468-aeb7-f3180cfb622f&#x27;, &#x27;name&#x27;: &#x27;proj:&#x27;, &#x27;description&#x27;: &#x27;Coordinate reference system information for geospatial data&#x27;, &#x27;spec_url&#x27;: &#x27;https://github.com/zarr-experimental/geo-proj/blob/v1/README.md&#x27;, &#x27;schema_url&#x27;: &#x27;https://raw.githubusercontent.com/zarr-experimental/geo-proj/refs/tags/v1/schema.json&#x27;}, {&#x27;uuid&#x27;: &#x27;689b58e2-cf7b-45e0-9fff-9cfc0883d6b4&#x27;, &#x27;name&#x27;: &#x27;spatial:&#x27;, &#x27;description&#x27;: &#x27;Spatial coordinate information&#x27;, &#x27;spec_url&#x27;: &#x27;https://github.com/zarr-conventions/spatial/blob/v1/README.md&#x27;, &#x27;schema_url&#x27;: &#x27;https://raw.githubusercontent.com/zarr-conventions/spatial/refs/tags/v1/schema.json&#x27;}]</dd><dt><span>_zarrs :</span></dt><dd>{&#x27;description&#x27;: &#x27;This array was created with zarrs&#x27;, &#x27;repository&#x27;: &#x27;https://github.com/zarrs/zarrs&#x27;, &#x27;version&#x27;: &#x27;0.23.0&#x27;}</dd></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray 'embeddings' (time: 7, embedding: 64, y: 1859584, x: 4009984)> Size: 3PB\n",
"[3340692134821888 values with dtype=int8]\n",
"Coordinates:\n",
" * time (time) int32 28B 2018 2019 2020 2021 2022 2023 2024\n",
" * y (y) float64 15MB 83.69 83.69 83.69 83.69 ... -83.36 -83.36 -83.36\n",
" * x (x) float64 32MB -180.0 -180.0 -180.0 -180.0 ... 180.2 180.2 180.2\n",
"Dimensions without coordinates: embedding\n",
"Attributes:\n",
" proj:code: EPSG:4326\n",
" spatial:dimensions: ['y', 'x']\n",
" spatial:transform: [8.983111749910169e-05, 0.0, -180.0, 0.0, -8.983...\n",
" spatial:transform_type: affine\n",
" spatial:bbox: [-180.0, -83.36280346631479, 180.2213438735178, ...\n",
" spatial:shape: [1859584, 4009984]\n",
" spatial:registration: pixel\n",
" embedding_dimensions: ['A00', 'A01', 'A02', 'A03', 'A04', 'A05', 'A06'...\n",
" zarr_conventions: [{'uuid': 'f17cb550-5864-4468-aeb7-f3180cfb622f'...\n",
" _zarrs: {'description': 'This array was created with zar..."
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"embedding_ds"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8f1810f7",
"metadata": {},
"outputs": [],
"source": [
"# Seattleish bbox\n",
"minx, miny, maxx, maxy = (-122.4440, 47.4949, -122.1941, 47.6515)"
]
},
{
"cell_type": "code",
"execution_count": 42,
"id": "e6c8c177",
"metadata": {},
"outputs": [],
"source": [
"t0 = time.time()\n",
"subset = embedding_ds.sel(\n",
" x=slice(minx, maxx),\n",
" y=slice(maxy, miny),\n",
" time=2020\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "7a131a03",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
"<defs>\n",
"<symbol id=\"icon-database\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M16 0c-8.837 0-16 2.239-16 5v4c0 2.761 7.163 5 16 5s16-2.239 16-5v-4c0-2.761-7.163-5-16-5z\"></path>\n",
"<path d=\"M16 17c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"<path d=\"M16 26c-8.837 0-16-2.239-16-5v6c0 2.761 7.163 5 16 5s16-2.239 16-5v-6c0 2.761-7.163 5-16 5z\"></path>\n",
"</symbol>\n",
"<symbol id=\"icon-file-text2\" viewBox=\"0 0 32 32\">\n",
"<path d=\"M28.681 7.159c-0.694-0.947-1.662-2.053-2.724-3.116s-2.169-2.030-3.116-2.724c-1.612-1.182-2.393-1.319-2.841-1.319h-15.5c-1.378 0-2.5 1.121-2.5 2.5v27c0 1.378 1.122 2.5 2.5 2.5h23c1.378 0 2.5-1.122 2.5-2.5v-19.5c0-0.448-0.137-1.23-1.319-2.841zM24.543 5.457c0.959 0.959 1.712 1.825 2.268 2.543h-4.811v-4.811c0.718 0.556 1.584 1.309 2.543 2.268zM28 29.5c0 0.271-0.229 0.5-0.5 0.5h-23c-0.271 0-0.5-0.229-0.5-0.5v-27c0-0.271 0.229-0.5 0.5-0.5 0 0 15.499-0 15.5 0v7c0 0.552 0.448 1 1 1h7v19.5z\"></path>\n",
"<path d=\"M23 26h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 22h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"<path d=\"M23 18h-14c-0.552 0-1-0.448-1-1s0.448-1 1-1h14c0.552 0 1 0.448 1 1s-0.448 1-1 1z\"></path>\n",
"</symbol>\n",
"</defs>\n",
"</svg>\n",
"<style>/* CSS stylesheet for displaying xarray objects in notebooks */\n",
"\n",
":root {\n",
" --xr-font-color0: var(\n",
" --jp-content-font-color0,\n",
" var(--pst-color-text-base rgba(0, 0, 0, 1))\n",
" );\n",
" --xr-font-color2: var(\n",
" --jp-content-font-color2,\n",
" var(--pst-color-text-base, rgba(0, 0, 0, 0.54))\n",
" );\n",
" --xr-font-color3: var(\n",
" --jp-content-font-color3,\n",
" var(--pst-color-text-base, rgba(0, 0, 0, 0.38))\n",
" );\n",
" --xr-border-color: var(\n",
" --jp-border-color2,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 10))\n",
" );\n",
" --xr-disabled-color: var(\n",
" --jp-layout-color3,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 40))\n",
" );\n",
" --xr-background-color: var(\n",
" --jp-layout-color0,\n",
" var(--pst-color-on-background, white)\n",
" );\n",
" --xr-background-color-row-even: var(\n",
" --jp-layout-color1,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 5))\n",
" );\n",
" --xr-background-color-row-odd: var(\n",
" --jp-layout-color2,\n",
" hsl(from var(--pst-color-on-background, white) h s calc(l - 15))\n",
" );\n",
"}\n",
"\n",
"html[theme=\"dark\"],\n",
"html[data-theme=\"dark\"],\n",
"body[data-theme=\"dark\"],\n",
"body.vscode-dark {\n",
" --xr-font-color0: var(\n",
" --jp-content-font-color0,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 1))\n",
" );\n",
" --xr-font-color2: var(\n",
" --jp-content-font-color2,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 0.54))\n",
" );\n",
" --xr-font-color3: var(\n",
" --jp-content-font-color3,\n",
" var(--pst-color-text-base, rgba(255, 255, 255, 0.38))\n",
" );\n",
" --xr-border-color: var(\n",
" --jp-border-color2,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 10))\n",
" );\n",
" --xr-disabled-color: var(\n",
" --jp-layout-color3,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 40))\n",
" );\n",
" --xr-background-color: var(\n",
" --jp-layout-color0,\n",
" var(--pst-color-on-background, #111111)\n",
" );\n",
" --xr-background-color-row-even: var(\n",
" --jp-layout-color1,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 5))\n",
" );\n",
" --xr-background-color-row-odd: var(\n",
" --jp-layout-color2,\n",
" hsl(from var(--pst-color-on-background, #111111) h s calc(l + 15))\n",
" );\n",
"}\n",
"\n",
".xr-wrap {\n",
" display: block !important;\n",
" min-width: 300px;\n",
" max-width: 700px;\n",
" line-height: 1.6;\n",
"}\n",
"\n",
".xr-text-repr-fallback {\n",
" /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
" display: none;\n",
"}\n",
"\n",
".xr-header {\n",
" padding-top: 6px;\n",
" padding-bottom: 6px;\n",
" margin-bottom: 4px;\n",
" border-bottom: solid 1px var(--xr-border-color);\n",
"}\n",
"\n",
".xr-header > div,\n",
".xr-header > ul {\n",
" display: inline;\n",
" margin-top: 0;\n",
" margin-bottom: 0;\n",
"}\n",
"\n",
".xr-obj-type,\n",
".xr-obj-name,\n",
".xr-group-name {\n",
" margin-left: 2px;\n",
" margin-right: 10px;\n",
"}\n",
"\n",
".xr-group-name::before {\n",
" content: \"📁\";\n",
" padding-right: 0.3em;\n",
"}\n",
"\n",
".xr-group-name,\n",
".xr-obj-type {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-sections {\n",
" padding-left: 0 !important;\n",
" display: grid;\n",
" grid-template-columns: 150px auto auto 1fr 0 20px 0 20px;\n",
" margin-block-start: 0;\n",
" margin-block-end: 0;\n",
"}\n",
"\n",
".xr-section-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-section-item input {\n",
" display: inline-block;\n",
" opacity: 0;\n",
" height: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-section-item input + label {\n",
" color: var(--xr-disabled-color);\n",
" border: 2px solid transparent !important;\n",
"}\n",
"\n",
".xr-section-item input:enabled + label {\n",
" cursor: pointer;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-item input:focus + label {\n",
" border: 2px solid var(--xr-font-color0) !important;\n",
"}\n",
"\n",
".xr-section-item input:enabled + label:hover {\n",
" color: var(--xr-font-color0);\n",
"}\n",
"\n",
".xr-section-summary {\n",
" grid-column: 1;\n",
" color: var(--xr-font-color2);\n",
" font-weight: 500;\n",
"}\n",
"\n",
".xr-section-summary > span {\n",
" display: inline-block;\n",
" padding-left: 0.5em;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label {\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-section-summary-in + label:before {\n",
" display: inline-block;\n",
" content: \"►\";\n",
" font-size: 11px;\n",
" width: 15px;\n",
" text-align: center;\n",
"}\n",
"\n",
".xr-section-summary-in:disabled + label:before {\n",
" color: var(--xr-disabled-color);\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label:before {\n",
" content: \"▼\";\n",
"}\n",
"\n",
".xr-section-summary-in:checked + label > span {\n",
" display: none;\n",
"}\n",
"\n",
".xr-section-summary,\n",
".xr-section-inline-details {\n",
" padding-top: 4px;\n",
"}\n",
"\n",
".xr-section-inline-details {\n",
" grid-column: 2 / -1;\n",
"}\n",
"\n",
".xr-section-details {\n",
" display: none;\n",
" grid-column: 1 / -1;\n",
" margin-top: 4px;\n",
" margin-bottom: 5px;\n",
"}\n",
"\n",
".xr-section-summary-in:checked ~ .xr-section-details {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-group-box {\n",
" display: inline-grid;\n",
" grid-template-columns: 0px 20px auto;\n",
" width: 100%;\n",
"}\n",
"\n",
".xr-group-box-vline {\n",
" grid-column-start: 1;\n",
" border-right: 0.2em solid;\n",
" border-color: var(--xr-border-color);\n",
" width: 0px;\n",
"}\n",
"\n",
".xr-group-box-hline {\n",
" grid-column-start: 2;\n",
" grid-row-start: 1;\n",
" height: 1em;\n",
" width: 20px;\n",
" border-bottom: 0.2em solid;\n",
" border-color: var(--xr-border-color);\n",
"}\n",
"\n",
".xr-group-box-contents {\n",
" grid-column-start: 3;\n",
"}\n",
"\n",
".xr-array-wrap {\n",
" grid-column: 1 / -1;\n",
" display: grid;\n",
" grid-template-columns: 20px auto;\n",
"}\n",
"\n",
".xr-array-wrap > label {\n",
" grid-column: 1;\n",
" vertical-align: top;\n",
"}\n",
"\n",
".xr-preview {\n",
" color: var(--xr-font-color3);\n",
"}\n",
"\n",
".xr-array-preview,\n",
".xr-array-data {\n",
" padding: 0 5px !important;\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-array-data,\n",
".xr-array-in:checked ~ .xr-array-preview {\n",
" display: none;\n",
"}\n",
"\n",
".xr-array-in:checked ~ .xr-array-data,\n",
".xr-array-preview {\n",
" display: inline-block;\n",
"}\n",
"\n",
".xr-dim-list {\n",
" display: inline-block !important;\n",
" list-style: none;\n",
" padding: 0 !important;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list li {\n",
" display: inline-block;\n",
" padding: 0;\n",
" margin: 0;\n",
"}\n",
"\n",
".xr-dim-list:before {\n",
" content: \"(\";\n",
"}\n",
"\n",
".xr-dim-list:after {\n",
" content: \")\";\n",
"}\n",
"\n",
".xr-dim-list li:not(:last-child):after {\n",
" content: \",\";\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-has-index {\n",
" font-weight: bold;\n",
"}\n",
"\n",
".xr-var-list,\n",
".xr-var-item {\n",
" display: contents;\n",
"}\n",
"\n",
".xr-var-item > div,\n",
".xr-var-item label,\n",
".xr-var-item > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-even);\n",
" border-color: var(--xr-background-color-row-odd);\n",
" margin-bottom: 0;\n",
" padding-top: 2px;\n",
"}\n",
"\n",
".xr-var-item > .xr-var-name:hover span {\n",
" padding-right: 5px;\n",
"}\n",
"\n",
".xr-var-list > li:nth-child(odd) > div,\n",
".xr-var-list > li:nth-child(odd) > label,\n",
".xr-var-list > li:nth-child(odd) > .xr-var-name span {\n",
" background-color: var(--xr-background-color-row-odd);\n",
" border-color: var(--xr-background-color-row-even);\n",
"}\n",
"\n",
".xr-var-name {\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-var-dims {\n",
" grid-column: 2;\n",
"}\n",
"\n",
".xr-var-dtype {\n",
" grid-column: 3;\n",
" text-align: right;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-preview {\n",
" grid-column: 4;\n",
"}\n",
"\n",
".xr-index-preview {\n",
" grid-column: 2 / 5;\n",
" color: var(--xr-font-color2);\n",
"}\n",
"\n",
".xr-var-name,\n",
".xr-var-dims,\n",
".xr-var-dtype,\n",
".xr-preview,\n",
".xr-attrs dt {\n",
" white-space: nowrap;\n",
" overflow: hidden;\n",
" text-overflow: ellipsis;\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-var-name:hover,\n",
".xr-var-dims:hover,\n",
".xr-var-dtype:hover,\n",
".xr-attrs dt:hover {\n",
" overflow: visible;\n",
" width: auto;\n",
" z-index: 1;\n",
"}\n",
"\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" display: none;\n",
" border-top: 2px dotted var(--xr-background-color);\n",
" padding-bottom: 20px !important;\n",
" padding-top: 10px !important;\n",
"}\n",
"\n",
".xr-var-attrs-in + label,\n",
".xr-var-data-in + label,\n",
".xr-index-data-in + label {\n",
" padding: 0 1px;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked ~ .xr-var-attrs,\n",
".xr-var-data-in:checked ~ .xr-var-data,\n",
".xr-index-data-in:checked ~ .xr-index-data {\n",
" display: block;\n",
"}\n",
"\n",
".xr-var-data > table {\n",
" float: right;\n",
"}\n",
"\n",
".xr-var-data > pre,\n",
".xr-index-data > pre,\n",
".xr-var-data > table > tbody > tr {\n",
" background-color: transparent !important;\n",
"}\n",
"\n",
".xr-var-name span,\n",
".xr-var-data,\n",
".xr-index-name div,\n",
".xr-index-data,\n",
".xr-attrs {\n",
" padding-left: 25px !important;\n",
"}\n",
"\n",
".xr-attrs,\n",
".xr-var-attrs,\n",
".xr-var-data,\n",
".xr-index-data {\n",
" grid-column: 1 / -1;\n",
"}\n",
"\n",
"dl.xr-attrs {\n",
" padding: 0;\n",
" margin: 0;\n",
" display: grid;\n",
" grid-template-columns: 125px auto;\n",
"}\n",
"\n",
".xr-attrs dt,\n",
".xr-attrs dd {\n",
" padding: 0;\n",
" margin: 0;\n",
" float: left;\n",
" padding-right: 10px;\n",
" width: auto;\n",
"}\n",
"\n",
".xr-attrs dt {\n",
" font-weight: normal;\n",
" grid-column: 1;\n",
"}\n",
"\n",
".xr-attrs dt:hover span {\n",
" display: inline-block;\n",
" background: var(--xr-background-color);\n",
" padding-right: 10px;\n",
"}\n",
"\n",
".xr-attrs dd {\n",
" grid-column: 2;\n",
" white-space: pre-wrap;\n",
" word-break: break-all;\n",
"}\n",
"\n",
".xr-icon-database,\n",
".xr-icon-file-text2,\n",
".xr-no-icon {\n",
" display: inline-block;\n",
" vertical-align: middle;\n",
" width: 1em;\n",
" height: 1.5em !important;\n",
" stroke-width: 0;\n",
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
"\n",
".xr-var-attrs-in:checked + label > .xr-icon-file-text2,\n",
".xr-var-data-in:checked + label > .xr-icon-database,\n",
".xr-index-data-in:checked + label > .xr-icon-database {\n",
" color: var(--xr-font-color0);\n",
" filter: drop-shadow(1px 1px 5px var(--xr-font-color2));\n",
" stroke-width: 0.8px;\n",
"}\n",
"</style><pre class='xr-text-repr-fallback'>&lt;xarray.DataArray &#x27;embeddings&#x27; (embedding: 64, y: 1743, x: 2782)&gt; Size: 310MB\n",
"[310337664 values with dtype=int8]\n",
"Coordinates:\n",
" * y (y) float64 14kB 47.65 47.65 47.65 47.65 ... 47.5 47.5 47.5 47.49\n",
" * x (x) float64 22kB -122.4 -122.4 -122.4 ... -122.2 -122.2 -122.2\n",
" time int32 4B 2020\n",
"Dimensions without coordinates: embedding\n",
"Attributes:\n",
" proj:code: EPSG:4326\n",
" spatial:dimensions: [&#x27;y&#x27;, &#x27;x&#x27;]\n",
" spatial:transform: [8.983111749910169e-05, 0.0, -180.0, 0.0, -8.983...\n",
" spatial:transform_type: affine\n",
" spatial:bbox: [-180.0, -83.36280346631479, 180.2213438735178, ...\n",
" spatial:shape: [1859584, 4009984]\n",
" spatial:registration: pixel\n",
" embedding_dimensions: [&#x27;A00&#x27;, &#x27;A01&#x27;, &#x27;A02&#x27;, &#x27;A03&#x27;, &#x27;A04&#x27;, &#x27;A05&#x27;, &#x27;A06&#x27;...\n",
" zarr_conventions: [{&#x27;uuid&#x27;: &#x27;f17cb550-5864-4468-aeb7-f3180cfb622f&#x27;...\n",
" _zarrs: {&#x27;description&#x27;: &#x27;This array was created with zar...</pre><div class='xr-wrap' style='display:none'><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-obj-name'>&#x27;embeddings&#x27;</div><ul class='xr-dim-list'><li><span>embedding</span>: 64</li><li><span class='xr-has-index'>y</span>: 1743</li><li><span class='xr-has-index'>x</span>: 2782</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-98362400-f877-4891-98b9-fd529f87da41' class='xr-array-in' type='checkbox' checked><label for='section-98362400-f877-4891-98b9-fd529f87da41' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>...</span></div><div class='xr-array-data'><pre>[310337664 values with dtype=int8]</pre></div></div></li><li class='xr-section-item'><input id='section-0b5b261d-eb07-4a23-813b-4373d848c6f9' class='xr-section-summary-in' type='checkbox' checked><label for='section-0b5b261d-eb07-4a23-813b-4373d848c6f9' class='xr-section-summary' >Coordinates: <span>(3)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>y</span></div><div class='xr-var-dims'>(y)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>47.65 47.65 47.65 ... 47.5 47.49</div><input id='attrs-5ee1d8a8-935f-4153-8bdb-3906868be59d' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-5ee1d8a8-935f-4153-8bdb-3906868be59d' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-10bfea68-d81f-43d8-ba00-82f4855e10bd' class='xr-var-data-in' type='checkbox'><label for='data-10bfea68-d81f-43d8-ba00-82f4855e10bd' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>latitude</dd><dt><span>long_name :</span></dt><dd>latitude coordinate</dd><dt><span>units :</span></dt><dd>degrees_north</dd><dt><span>axis :</span></dt><dd>Y</dd></dl></div><div class='xr-var-data'><pre>array([47.651435, 47.651345, 47.651255, ..., 47.495129, 47.495039, 47.494949],\n",
" shape=(1743,))</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>x</span></div><div class='xr-var-dims'>(x)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>-122.4 -122.4 ... -122.2 -122.2</div><input id='attrs-59596cc8-709c-439b-856d-62e9f6dfca58' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-59596cc8-709c-439b-856d-62e9f6dfca58' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-dec049df-a355-4d12-b320-34bd23dbe651' class='xr-var-data-in' type='checkbox'><label for='data-dec049df-a355-4d12-b320-34bd23dbe651' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>longitude</dd><dt><span>long_name :</span></dt><dd>longitude coordinate</dd><dt><span>units :</span></dt><dd>degrees_east</dd><dt><span>axis :</span></dt><dd>X</dd></dl></div><div class='xr-var-data'><pre>array([-122.44399 , -122.4439 , -122.443811, ..., -122.19435 , -122.19426 ,\n",
" -122.19417 ], shape=(2782,))</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>time</span></div><div class='xr-var-dims'>()</div><div class='xr-var-dtype'>int32</div><div class='xr-var-preview xr-preview'>2020</div><input id='attrs-b5ce5e48-8c77-4a93-825f-86b7be03df08' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-b5ce5e48-8c77-4a93-825f-86b7be03df08' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-da02d030-8eba-4e35-84d0-f66e8ae6f3c6' class='xr-var-data-in' type='checkbox'><label for='data-da02d030-8eba-4e35-84d0-f66e8ae6f3c6' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>_zarrs :</span></dt><dd>{&#x27;description&#x27;: &#x27;This array was created with zarrs&#x27;, &#x27;repository&#x27;: &#x27;https://github.com/zarrs/zarrs&#x27;, &#x27;version&#x27;: &#x27;0.23.0&#x27;}</dd></dl></div><div class='xr-var-data'><pre>array(2020, dtype=int32)</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-e4327383-8308-4d95-9b83-b0d6ceba88f6' class='xr-section-summary-in' type='checkbox' ><label for='section-e4327383-8308-4d95-9b83-b0d6ceba88f6' class='xr-section-summary' >Attributes: <span>(10)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>proj:code :</span></dt><dd>EPSG:4326</dd><dt><span>spatial:dimensions :</span></dt><dd>[&#x27;y&#x27;, &#x27;x&#x27;]</dd><dt><span>spatial:transform :</span></dt><dd>[8.983111749910169e-05, 0.0, -180.0, 0.0, -8.983111749910169e-05, 83.68570533713473]</dd><dt><span>spatial:transform_type :</span></dt><dd>affine</dd><dt><span>spatial:bbox :</span></dt><dd>[-180.0, -83.36280346631479, 180.2213438735178, 83.68570533713473]</dd><dt><span>spatial:shape :</span></dt><dd>[1859584, 4009984]</dd><dt><span>spatial:registration :</span></dt><dd>pixel</dd><dt><span>embedding_dimensions :</span></dt><dd>[&#x27;A00&#x27;, &#x27;A01&#x27;, &#x27;A02&#x27;, &#x27;A03&#x27;, &#x27;A04&#x27;, &#x27;A05&#x27;, &#x27;A06&#x27;, &#x27;A07&#x27;, &#x27;A08&#x27;, &#x27;A09&#x27;, &#x27;A10&#x27;, &#x27;A11&#x27;, &#x27;A12&#x27;, &#x27;A13&#x27;, &#x27;A14&#x27;, &#x27;A15&#x27;, &#x27;A16&#x27;, &#x27;A17&#x27;, &#x27;A18&#x27;, &#x27;A19&#x27;, &#x27;A20&#x27;, &#x27;A21&#x27;, &#x27;A22&#x27;, &#x27;A23&#x27;, &#x27;A24&#x27;, &#x27;A25&#x27;, &#x27;A26&#x27;, &#x27;A27&#x27;, &#x27;A28&#x27;, &#x27;A29&#x27;, &#x27;A30&#x27;, &#x27;A31&#x27;, &#x27;A32&#x27;, &#x27;A33&#x27;, &#x27;A34&#x27;, &#x27;A35&#x27;, &#x27;A36&#x27;, &#x27;A37&#x27;, &#x27;A38&#x27;, &#x27;A39&#x27;, &#x27;A40&#x27;, &#x27;A41&#x27;, &#x27;A42&#x27;, &#x27;A43&#x27;, &#x27;A44&#x27;, &#x27;A45&#x27;, &#x27;A46&#x27;, &#x27;A47&#x27;, &#x27;A48&#x27;, &#x27;A49&#x27;, &#x27;A50&#x27;, &#x27;A51&#x27;, &#x27;A52&#x27;, &#x27;A53&#x27;, &#x27;A54&#x27;, &#x27;A55&#x27;, &#x27;A56&#x27;, &#x27;A57&#x27;, &#x27;A58&#x27;, &#x27;A59&#x27;, &#x27;A60&#x27;, &#x27;A61&#x27;, &#x27;A62&#x27;, &#x27;A63&#x27;]</dd><dt><span>zarr_conventions :</span></dt><dd>[{&#x27;uuid&#x27;: &#x27;f17cb550-5864-4468-aeb7-f3180cfb622f&#x27;, &#x27;name&#x27;: &#x27;proj:&#x27;, &#x27;description&#x27;: &#x27;Coordinate reference system information for geospatial data&#x27;, &#x27;spec_url&#x27;: &#x27;https://github.com/zarr-experimental/geo-proj/blob/v1/README.md&#x27;, &#x27;schema_url&#x27;: &#x27;https://raw.githubusercontent.com/zarr-experimental/geo-proj/refs/tags/v1/schema.json&#x27;}, {&#x27;uuid&#x27;: &#x27;689b58e2-cf7b-45e0-9fff-9cfc0883d6b4&#x27;, &#x27;name&#x27;: &#x27;spatial:&#x27;, &#x27;description&#x27;: &#x27;Spatial coordinate information&#x27;, &#x27;spec_url&#x27;: &#x27;https://github.com/zarr-conventions/spatial/blob/v1/README.md&#x27;, &#x27;schema_url&#x27;: &#x27;https://raw.githubusercontent.com/zarr-conventions/spatial/refs/tags/v1/schema.json&#x27;}]</dd><dt><span>_zarrs :</span></dt><dd>{&#x27;description&#x27;: &#x27;This array was created with zarrs&#x27;, &#x27;repository&#x27;: &#x27;https://github.com/zarrs/zarrs&#x27;, &#x27;version&#x27;: &#x27;0.23.0&#x27;}</dd></dl></div></li></ul></div></div>"
],
"text/plain": [
"<xarray.DataArray 'embeddings' (embedding: 64, y: 1743, x: 2782)> Size: 310MB\n",
"[310337664 values with dtype=int8]\n",
"Coordinates:\n",
" * y (y) float64 14kB 47.65 47.65 47.65 47.65 ... 47.5 47.5 47.5 47.49\n",
" * x (x) float64 22kB -122.4 -122.4 -122.4 ... -122.2 -122.2 -122.2\n",
" time int32 4B 2020\n",
"Dimensions without coordinates: embedding\n",
"Attributes:\n",
" proj:code: EPSG:4326\n",
" spatial:dimensions: ['y', 'x']\n",
" spatial:transform: [8.983111749910169e-05, 0.0, -180.0, 0.0, -8.983...\n",
" spatial:transform_type: affine\n",
" spatial:bbox: [-180.0, -83.36280346631479, 180.2213438735178, ...\n",
" spatial:shape: [1859584, 4009984]\n",
" spatial:registration: pixel\n",
" embedding_dimensions: ['A00', 'A01', 'A02', 'A03', 'A04', 'A05', 'A06'...\n",
" zarr_conventions: [{'uuid': 'f17cb550-5864-4468-aeb7-f3180cfb622f'...\n",
" _zarrs: {'description': 'This array was created with zar..."
]
},
"execution_count": 43,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"subset"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "1698fbc9",
"metadata": {},
"outputs": [],
"source": [
"data = subset.compute().values"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "30272f47",
"metadata": {},
"outputs": [],
"source": [
"num_channels, height, width = data.shape\n",
"\n",
"# pca transform\n",
"n_components = 3\n",
"pca = PCA(n_components=n_components)\n",
"reshaped_data = data.reshape(num_channels, -1).T\n",
"pca.fit(reshaped_data[::10])\n",
"pca_transformed = pca.transform(reshaped_data)\n",
"pca_transformed = pca_transformed.reshape(height, width, n_components)"
]
},
{
"cell_type": "code",
"execution_count": 59,
"id": "86f7ee12",
"metadata": {},
"outputs": [],
"source": [
"# percentile normalize\n",
"pca_transformed = (pca_transformed - np.percentile(pca_transformed, 1, axis=(0, 1)))\n",
"pca_transformed = pca_transformed / np.percentile(pca_transformed, 99, axis=(0, 1))\n",
"pca_transformed = np.clip(pca_transformed, 0, 1)"
]
},
{
"cell_type": "code",
"execution_count": 60,
"id": "5e94edb3",
"metadata": {},
"outputs": [
{
"data": {
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment