Skip to content

Instantly share code, notes, and snippets.

@kif

kif/bliss-chunk.ipynb

Last active October 22, 2021 09:37
Show Gist options
  • Save kif/dfb190cf8fe7179b7b773bde39195391 to your computer and use it in GitHub Desktop.
Save kif/dfb190cf8fe7179b7b773bde39195391 to your computer and use it in GitHub Desktop.
Download ZIP
Some insights about chunking & compression for MCA data in BLISS
Display the source blob
Display the rendered blob
Raw
bliss-chunk.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "ultimate-indonesian",
"metadata": {},
"source": [
"# Chuncking of dataset containing 1D data\n",
"\n",
"Case study for the storage of MCA and other ROI-collection in Bliss. Inspired by the ROI-collection of ESRF-ID22\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "indirect-mounting",
"metadata": {},
"outputs": [],
"source": [
"%matplotlib nbagg"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "prescription-animal",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"h5py v3.2.1. HDF5 v1.12.0. hdf5plugin v3.2.0\n",
"HDF5PluginBuildOptions(openmp=True, native=True, sse2=True, avx2=True, cpp11=True, filter_file_extension='.so')\n",
"{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}\n"
]
}
],
"source": [
"import tempfile\n",
"import os\n",
"import time\n",
"from collections import namedtuple\n",
"import numpy\n",
"import h5py\n",
"import hdf5plugin\n",
"from matplotlib.pyplot import subplots\n",
"\n",
"print(f\"h5py v{h5py.version.version}. HDF5 v{h5py.version.hdf5_version}. hdf5plugin v{hdf5plugin.version}\")\n",
"print(hdf5plugin.config)\n",
"print(os.sched_getaffinity(0))\n",
"\n",
"Key = namedtuple(\"Key\", \"format compression chunk\")\n",
"Res = namedtuple(\"Res\", \"compression read write\")\n",
"\n",
"M = 1<<20\n",
"\n",
"fname = \"/mnt/data/ID22/MultiAnalyzer/LaB6_35keV_mantr10_nozzle20_1/LaB6_35keV_mantr10_nozzle20_1_0001/LaB6_35keV_mantr10_nozzle20_1_0001.h5\"\n",
"hpath = \"/1.1/measurement/eiger_roi_collection\""
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "antique-spouse",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Architecture : x86_64\r\n",
"Mode(s) opératoire(s) des processeurs : 32-bit, 64-bit\r\n",
"Boutisme : Little Endian\r\n",
"Tailles des adresses: 43 bits physical, 48 bits virtual\r\n",
"Processeur(s) : 16\r\n",
"Liste de processeur(s) en ligne : 0-15\r\n",
"Thread(s) par cœur : 2\r\n",
"Cœur(s) par socket : 8\r\n",
"Socket(s) : 1\r\n",
"Nœud(s) NUMA : 1\r\n",
"Identifiant constructeur : AuthenticAMD\r\n",
"Famille de processeur : 23\r\n",
"Modèle : 49\r\n",
"Nom de modèle : AMD EPYC 7262 8-Core Processor\r\n",
"Révision : 0\r\n",
"Accroissement de fréquence : activé\r\n",
"Vitesse du processeur en MHz : 2498.267\r\n",
"Vitesse maximale du processeur en MHz : 3400,0000\r\n",
"Vitesse minimale du processeur en MHz : 1500,0000\r\n",
"BogoMIPS : 6400.27\r\n",
"Virtualisation : AMD-V\r\n",
"Cache L1d : 256 KiB\r\n",
"Cache L1i : 256 KiB\r\n",
"Cache L2 : 4 MiB\r\n",
"Cache L3 : 128 MiB\r\n",
"Nœud NUMA 0 de processeur(s) : 0-15\r\n"
]
}
],
"source": [
"!lscpu |head -n 26"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "corporate-resource",
"metadata": {},
"outputs": [],
"source": [
"t0 = time.perf_counter()\n",
"with h5py.File(fname) as h:\n",
" ds = h[hpath][()]\n",
"dt = time.perf_counter()-t0"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "conscious-ladder",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dataset of shape (390000, 6656) with dtype: int32, total size of 9902.344 MB\n",
"Read time (from NFS): 34.774s, Read speed: 284.765 MB/s\n",
"This is a very compressible dataset with a maximum value of 260\n"
]
},
{
"data": {
"application/javascript": [
"/* Put everything inside the global mpl namespace */\n",
"/* global mpl */\n",
"window.mpl = {};\n",
"\n",
"mpl.get_websocket_type = function () {\n",
" if (typeof WebSocket !== 'undefined') {\n",
" return WebSocket;\n",
" } else if (typeof MozWebSocket !== 'undefined') {\n",
" return MozWebSocket;\n",
" } else {\n",
" alert(\n",
" 'Your browser does not have WebSocket support. ' +\n",
" 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
" 'Firefox 4 and 5 are also supported but you ' +\n",
" 'have to enable WebSockets in about:config.'\n",
" );\n",
" }\n",
"};\n",
"\n",
"mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n",
" this.id = figure_id;\n",
"\n",
" this.ws = websocket;\n",
"\n",
" this.supports_binary = this.ws.binaryType !== undefined;\n",
"\n",
" if (!this.supports_binary) {\n",
" var warnings = document.getElementById('mpl-warnings');\n",
" if (warnings) {\n",
" warnings.style.display = 'block';\n",
" warnings.textContent =\n",
" 'This browser does not support binary websocket messages. ' +\n",
" 'Performance may be slow.';\n",
" }\n",
" }\n",
"\n",
" this.imageObj = new Image();\n",
"\n",
" this.context = undefined;\n",
" this.message = undefined;\n",
" this.canvas = undefined;\n",
" this.rubberband_canvas = undefined;\n",
" this.rubberband_context = undefined;\n",
" this.format_dropdown = undefined;\n",
"\n",
" this.image_mode = 'full';\n",
"\n",
" this.root = document.createElement('div');\n",
" this.root.setAttribute('style', 'display: inline-block');\n",
" this._root_extra_style(this.root);\n",
"\n",
" parent_element.appendChild(this.root);\n",
"\n",
" this._init_header(this);\n",
" this._init_canvas(this);\n",
" this._init_toolbar(this);\n",
"\n",
" var fig = this;\n",
"\n",
" this.waiting = false;\n",
"\n",
" this.ws.onopen = function () {\n",
" fig.send_message('supports_binary', { value: fig.supports_binary });\n",
" fig.send_message('send_image_mode', {});\n",
" if (fig.ratio !== 1) {\n",
" fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n",
" }\n",
" fig.send_message('refresh', {});\n",
" };\n",
"\n",
" this.imageObj.onload = function () {\n",
" if (fig.image_mode === 'full') {\n",
" // Full images could contain transparency (where diff images\n",
" // almost always do), so we need to clear the canvas so that\n",
" // there is no ghosting.\n",
" fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
" }\n",
" fig.context.drawImage(fig.imageObj, 0, 0);\n",
" };\n",
"\n",
" this.imageObj.onunload = function () {\n",
" fig.ws.close();\n",
" };\n",
"\n",
" this.ws.onmessage = this._make_on_message_function(this);\n",
"\n",
" this.ondownload = ondownload;\n",
"};\n",
"\n",
"mpl.figure.prototype._init_header = function () {\n",
" var titlebar = document.createElement('div');\n",
" titlebar.classList =\n",
" 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n",
" var titletext = document.createElement('div');\n",
" titletext.classList = 'ui-dialog-title';\n",
" titletext.setAttribute(\n",
" 'style',\n",
" 'width: 100%; text-align: center; padding: 3px;'\n",
" );\n",
" titlebar.appendChild(titletext);\n",
" this.root.appendChild(titlebar);\n",
" this.header = titletext;\n",
"};\n",
"\n",
"mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n",
"\n",
"mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n",
"\n",
"mpl.figure.prototype._init_canvas = function () {\n",
" var fig = this;\n",
"\n",
" var canvas_div = (this.canvas_div = document.createElement('div'));\n",
" canvas_div.setAttribute(\n",
" 'style',\n",
" 'border: 1px solid #ddd;' +\n",
" 'box-sizing: content-box;' +\n",
" 'clear: both;' +\n",
" 'min-height: 1px;' +\n",
" 'min-width: 1px;' +\n",
" 'outline: 0;' +\n",
" 'overflow: hidden;' +\n",
" 'position: relative;' +\n",
" 'resize: both;'\n",
" );\n",
"\n",
" function on_keyboard_event_closure(name) {\n",
" return function (event) {\n",
" return fig.key_event(event, name);\n",
" };\n",
" }\n",
"\n",
" canvas_div.addEventListener(\n",
" 'keydown',\n",
" on_keyboard_event_closure('key_press')\n",
" );\n",
" canvas_div.addEventListener(\n",
" 'keyup',\n",
" on_keyboard_event_closure('key_release')\n",
" );\n",
"\n",
" this._canvas_extra_style(canvas_div);\n",
" this.root.appendChild(canvas_div);\n",
"\n",
" var canvas = (this.canvas = document.createElement('canvas'));\n",
" canvas.classList.add('mpl-canvas');\n",
" canvas.setAttribute('style', 'box-sizing: content-box;');\n",
"\n",
" this.context = canvas.getContext('2d');\n",
"\n",
" var backingStore =\n",
" this.context.backingStorePixelRatio ||\n",
" this.context.webkitBackingStorePixelRatio ||\n",
" this.context.mozBackingStorePixelRatio ||\n",
" this.context.msBackingStorePixelRatio ||\n",
" this.context.oBackingStorePixelRatio ||\n",
" this.context.backingStorePixelRatio ||\n",
" 1;\n",
"\n",
" this.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
"\n",
" var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n",
" 'canvas'\n",
" ));\n",
" rubberband_canvas.setAttribute(\n",
" 'style',\n",
" 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n",
" );\n",
"\n",
" // Apply a ponyfill if ResizeObserver is not implemented by browser.\n",
" if (this.ResizeObserver === undefined) {\n",
" if (window.ResizeObserver !== undefined) {\n",
" this.ResizeObserver = window.ResizeObserver;\n",
" } else {\n",
" var obs = _JSXTOOLS_RESIZE_OBSERVER({});\n",
" this.ResizeObserver = obs.ResizeObserver;\n",
" }\n",
" }\n",
"\n",
" this.resizeObserverInstance = new this.ResizeObserver(function (entries) {\n",
" var nentries = entries.length;\n",
" for (var i = 0; i < nentries; i++) {\n",
" var entry = entries[i];\n",
" var width, height;\n",
" if (entry.contentBoxSize) {\n",
" if (entry.contentBoxSize instanceof Array) {\n",
" // Chrome 84 implements new version of spec.\n",
" width = entry.contentBoxSize[0].inlineSize;\n",
" height = entry.contentBoxSize[0].blockSize;\n",
" } else {\n",
" // Firefox implements old version of spec.\n",
" width = entry.contentBoxSize.inlineSize;\n",
" height = entry.contentBoxSize.blockSize;\n",
" }\n",
" } else {\n",
" // Chrome <84 implements even older version of spec.\n",
" width = entry.contentRect.width;\n",
" height = entry.contentRect.height;\n",
" }\n",
"\n",
" // Keep the size of the canvas and rubber band canvas in sync with\n",
" // the canvas container.\n",
" if (entry.devicePixelContentBoxSize) {\n",
" // Chrome 84 implements new version of spec.\n",
" canvas.setAttribute(\n",
" 'width',\n",
" entry.devicePixelContentBoxSize[0].inlineSize\n",
" );\n",
" canvas.setAttribute(\n",
" 'height',\n",
" entry.devicePixelContentBoxSize[0].blockSize\n",
" );\n",
" } else {\n",
" canvas.setAttribute('width', width * fig.ratio);\n",
" canvas.setAttribute('height', height * fig.ratio);\n",
" }\n",
" canvas.setAttribute(\n",
" 'style',\n",
" 'width: ' + width + 'px; height: ' + height + 'px;'\n",
" );\n",
"\n",
" rubberband_canvas.setAttribute('width', width);\n",
" rubberband_canvas.setAttribute('height', height);\n",
"\n",
" // And update the size in Python. We ignore the initial 0/0 size\n",
" // that occurs as the element is placed into the DOM, which should\n",
" // otherwise not happen due to the minimum size styling.\n",
" if (fig.ws.readyState == 1 && width != 0 && height != 0) {\n",
" fig.request_resize(width, height);\n",
" }\n",
" }\n",
" });\n",
" this.resizeObserverInstance.observe(canvas_div);\n",
"\n",
" function on_mouse_event_closure(name) {\n",
" return function (event) {\n",
" return fig.mouse_event(event, name);\n",
" };\n",
" }\n",
"\n",
" rubberband_canvas.addEventListener(\n",
" 'mousedown',\n",
" on_mouse_event_closure('button_press')\n",
" );\n",
" rubberband_canvas.addEventListener(\n",
" 'mouseup',\n",
" on_mouse_event_closure('button_release')\n",
" );\n",
" rubberband_canvas.addEventListener(\n",
" 'dblclick',\n",
" on_mouse_event_closure('dblclick')\n",
" );\n",
" // Throttle sequential mouse events to 1 every 20ms.\n",
" rubberband_canvas.addEventListener(\n",
" 'mousemove',\n",
" on_mouse_event_closure('motion_notify')\n",
" );\n",
"\n",
" rubberband_canvas.addEventListener(\n",
" 'mouseenter',\n",
" on_mouse_event_closure('figure_enter')\n",
" );\n",
" rubberband_canvas.addEventListener(\n",
" 'mouseleave',\n",
" on_mouse_event_closure('figure_leave')\n",
" );\n",
"\n",
" canvas_div.addEventListener('wheel', function (event) {\n",
" if (event.deltaY < 0) {\n",
" event.step = 1;\n",
" } else {\n",
" event.step = -1;\n",
" }\n",
" on_mouse_event_closure('scroll')(event);\n",
" });\n",
"\n",
" canvas_div.appendChild(canvas);\n",
" canvas_div.appendChild(rubberband_canvas);\n",
"\n",
" this.rubberband_context = rubberband_canvas.getContext('2d');\n",
" this.rubberband_context.strokeStyle = '#000000';\n",
"\n",
" this._resize_canvas = function (width, height, forward) {\n",
" if (forward) {\n",
" canvas_div.style.width = width + 'px';\n",
" canvas_div.style.height = height + 'px';\n",
" }\n",
" };\n",
"\n",
" // Disable right mouse context menu.\n",
" this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n",
" event.preventDefault();\n",
" return false;\n",
" });\n",
"\n",
" function set_focus() {\n",
" canvas.focus();\n",
" canvas_div.focus();\n",
" }\n",
"\n",
" window.setTimeout(set_focus, 100);\n",
"};\n",
"\n",
"mpl.figure.prototype._init_toolbar = function () {\n",
" var fig = this;\n",
"\n",
" var toolbar = document.createElement('div');\n",
" toolbar.classList = 'mpl-toolbar';\n",
" this.root.appendChild(toolbar);\n",
"\n",
" function on_click_closure(name) {\n",
" return function (_event) {\n",
" return fig.toolbar_button_onclick(name);\n",
" };\n",
" }\n",
"\n",
" function on_mouseover_closure(tooltip) {\n",
" return function (event) {\n",
" if (!event.currentTarget.disabled) {\n",
" return fig.toolbar_button_onmouseover(tooltip);\n",
" }\n",
" };\n",
" }\n",
"\n",
" fig.buttons = {};\n",
" var buttonGroup = document.createElement('div');\n",
" buttonGroup.classList = 'mpl-button-group';\n",
" for (var toolbar_ind in mpl.toolbar_items) {\n",
" var name = mpl.toolbar_items[toolbar_ind][0];\n",
" var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
" var image = mpl.toolbar_items[toolbar_ind][2];\n",
" var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
"\n",
" if (!name) {\n",
" /* Instead of a spacer, we start a new button group. */\n",
" if (buttonGroup.hasChildNodes()) {\n",
" toolbar.appendChild(buttonGroup);\n",
" }\n",
" buttonGroup = document.createElement('div');\n",
" buttonGroup.classList = 'mpl-button-group';\n",
" continue;\n",
" }\n",
"\n",
" var button = (fig.buttons[name] = document.createElement('button'));\n",
" button.classList = 'mpl-widget';\n",
" button.setAttribute('role', 'button');\n",
" button.setAttribute('aria-disabled', 'false');\n",
" button.addEventListener('click', on_click_closure(method_name));\n",
" button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
"\n",
" var icon_img = document.createElement('img');\n",
" icon_img.src = '_images/' + image + '.png';\n",
" icon_img.srcset = '_images/' + image + '_large.png 2x';\n",
" icon_img.alt = tooltip;\n",
" button.appendChild(icon_img);\n",
"\n",
" buttonGroup.appendChild(button);\n",
" }\n",
"\n",
" if (buttonGroup.hasChildNodes()) {\n",
" toolbar.appendChild(buttonGroup);\n",
" }\n",
"\n",
" var fmt_picker = document.createElement('select');\n",
" fmt_picker.classList = 'mpl-widget';\n",
" toolbar.appendChild(fmt_picker);\n",
" this.format_dropdown = fmt_picker;\n",
"\n",
" for (var ind in mpl.extensions) {\n",
" var fmt = mpl.extensions[ind];\n",
" var option = document.createElement('option');\n",
" option.selected = fmt === mpl.default_extension;\n",
" option.innerHTML = fmt;\n",
" fmt_picker.appendChild(option);\n",
" }\n",
"\n",
" var status_bar = document.createElement('span');\n",
" status_bar.classList = 'mpl-message';\n",
" toolbar.appendChild(status_bar);\n",
" this.message = status_bar;\n",
"};\n",
"\n",
"mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n",
" // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
" // which will in turn request a refresh of the image.\n",
" this.send_message('resize', { width: x_pixels, height: y_pixels });\n",
"};\n",
"\n",
"mpl.figure.prototype.send_message = function (type, properties) {\n",
" properties['type'] = type;\n",
" properties['figure_id'] = this.id;\n",
" this.ws.send(JSON.stringify(properties));\n",
"};\n",
"\n",
"mpl.figure.prototype.send_draw_message = function () {\n",
" if (!this.waiting) {\n",
" this.waiting = true;\n",
" this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n",
" }\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
" var format_dropdown = fig.format_dropdown;\n",
" var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
" fig.ondownload(fig, format);\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_resize = function (fig, msg) {\n",
" var size = msg['size'];\n",
" if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n",
" fig._resize_canvas(size[0], size[1], msg['forward']);\n",
" fig.send_message('refresh', {});\n",
" }\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n",
" var x0 = msg['x0'] / fig.ratio;\n",
" var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n",
" var x1 = msg['x1'] / fig.ratio;\n",
" var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n",
" x0 = Math.floor(x0) + 0.5;\n",
" y0 = Math.floor(y0) + 0.5;\n",
" x1 = Math.floor(x1) + 0.5;\n",
" y1 = Math.floor(y1) + 0.5;\n",
" var min_x = Math.min(x0, x1);\n",
" var min_y = Math.min(y0, y1);\n",
" var width = Math.abs(x1 - x0);\n",
" var height = Math.abs(y1 - y0);\n",
"\n",
" fig.rubberband_context.clearRect(\n",
" 0,\n",
" 0,\n",
" fig.canvas.width / fig.ratio,\n",
" fig.canvas.height / fig.ratio\n",
" );\n",
"\n",
" fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n",
" // Updates the figure title.\n",
" fig.header.textContent = msg['label'];\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_cursor = function (fig, msg) {\n",
" var cursor = msg['cursor'];\n",
" switch (cursor) {\n",
" case 0:\n",
" cursor = 'pointer';\n",
" break;\n",
" case 1:\n",
" cursor = 'default';\n",
" break;\n",
" case 2:\n",
" cursor = 'crosshair';\n",
" break;\n",
" case 3:\n",
" cursor = 'move';\n",
" break;\n",
" }\n",
" fig.rubberband_canvas.style.cursor = cursor;\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_message = function (fig, msg) {\n",
" fig.message.textContent = msg['message'];\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_draw = function (fig, _msg) {\n",
" // Request the server to send over a new figure.\n",
" fig.send_draw_message();\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n",
" fig.image_mode = msg['mode'];\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n",
" for (var key in msg) {\n",
" if (!(key in fig.buttons)) {\n",
" continue;\n",
" }\n",
" fig.buttons[key].disabled = !msg[key];\n",
" fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n",
" }\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n",
" if (msg['mode'] === 'PAN') {\n",
" fig.buttons['Pan'].classList.add('active');\n",
" fig.buttons['Zoom'].classList.remove('active');\n",
" } else if (msg['mode'] === 'ZOOM') {\n",
" fig.buttons['Pan'].classList.remove('active');\n",
" fig.buttons['Zoom'].classList.add('active');\n",
" } else {\n",
" fig.buttons['Pan'].classList.remove('active');\n",
" fig.buttons['Zoom'].classList.remove('active');\n",
" }\n",
"};\n",
"\n",
"mpl.figure.prototype.updated_canvas_event = function () {\n",
" // Called whenever the canvas gets updated.\n",
" this.send_message('ack', {});\n",
"};\n",
"\n",
"// A function to construct a web socket function for onmessage handling.\n",
"// Called in the figure constructor.\n",
"mpl.figure.prototype._make_on_message_function = function (fig) {\n",
" return function socket_on_message(evt) {\n",
" if (evt.data instanceof Blob) {\n",
" var img = evt.data;\n",
" if (img.type !== 'image/png') {\n",
" /* FIXME: We get \"Resource interpreted as Image but\n",
" * transferred with MIME type text/plain:\" errors on\n",
" * Chrome. But how to set the MIME type? It doesn't seem\n",
" * to be part of the websocket stream */\n",
" img.type = 'image/png';\n",
" }\n",
"\n",
" /* Free the memory for the previous frames */\n",
" if (fig.imageObj.src) {\n",
" (window.URL || window.webkitURL).revokeObjectURL(\n",
" fig.imageObj.src\n",
" );\n",
" }\n",
"\n",
" fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
" img\n",
" );\n",
" fig.updated_canvas_event();\n",
" fig.waiting = false;\n",
" return;\n",
" } else if (\n",
" typeof evt.data === 'string' &&\n",
" evt.data.slice(0, 21) === 'data:image/png;base64'\n",
" ) {\n",
" fig.imageObj.src = evt.data;\n",
" fig.updated_canvas_event();\n",
" fig.waiting = false;\n",
" return;\n",
" }\n",
"\n",
" var msg = JSON.parse(evt.data);\n",
" var msg_type = msg['type'];\n",
"\n",
" // Call the \"handle_{type}\" callback, which takes\n",
" // the figure and JSON message as its only arguments.\n",
" try {\n",
" var callback = fig['handle_' + msg_type];\n",
" } catch (e) {\n",
" console.log(\n",
" \"No handler for the '\" + msg_type + \"' message type: \",\n",
" msg\n",
" );\n",
" return;\n",
" }\n",
"\n",
" if (callback) {\n",
" try {\n",
" // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
" callback(fig, msg);\n",
" } catch (e) {\n",
" console.log(\n",
" \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n",
" e,\n",
" e.stack,\n",
" msg\n",
" );\n",
" }\n",
" }\n",
" };\n",
"};\n",
"\n",
"// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
"mpl.findpos = function (e) {\n",
" //this section is from http://www.quirksmode.org/js/events_properties.html\n",
" var targ;\n",
" if (!e) {\n",
" e = window.event;\n",
" }\n",
" if (e.target) {\n",
" targ = e.target;\n",
" } else if (e.srcElement) {\n",
" targ = e.srcElement;\n",
" }\n",
" if (targ.nodeType === 3) {\n",
" // defeat Safari bug\n",
" targ = targ.parentNode;\n",
" }\n",
"\n",
" // pageX,Y are the mouse positions relative to the document\n",
" var boundingRect = targ.getBoundingClientRect();\n",
" var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n",
" var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n",
"\n",
" return { x: x, y: y };\n",
"};\n",
"\n",
"/*\n",
" * return a copy of an object with only non-object keys\n",
" * we need this to avoid circular references\n",
" * http://stackoverflow.com/a/24161582/3208463\n",
" */\n",
"function simpleKeys(original) {\n",
" return Object.keys(original).reduce(function (obj, key) {\n",
" if (typeof original[key] !== 'object') {\n",
" obj[key] = original[key];\n",
" }\n",
" return obj;\n",
" }, {});\n",
"}\n",
"\n",
"mpl.figure.prototype.mouse_event = function (event, name) {\n",
" var canvas_pos = mpl.findpos(event);\n",
"\n",
" if (name === 'button_press') {\n",
" this.canvas.focus();\n",
" this.canvas_div.focus();\n",
" }\n",
"\n",
" var x = canvas_pos.x * this.ratio;\n",
" var y = canvas_pos.y * this.ratio;\n",
"\n",
" this.send_message(name, {\n",
" x: x,\n",
" y: y,\n",
" button: event.button,\n",
" step: event.step,\n",
" guiEvent: simpleKeys(event),\n",
" });\n",
"\n",
" /* This prevents the web browser from automatically changing to\n",
" * the text insertion cursor when the button is pressed. We want\n",
" * to control all of the cursor setting manually through the\n",
" * 'cursor' event from matplotlib */\n",
" event.preventDefault();\n",
" return false;\n",
"};\n",
"\n",
"mpl.figure.prototype._key_event_extra = function (_event, _name) {\n",
" // Handle any extra behaviour associated with a key event\n",
"};\n",
"\n",
"mpl.figure.prototype.key_event = function (event, name) {\n",
" // Prevent repeat events\n",
" if (name === 'key_press') {\n",
" if (event.key === this._key) {\n",
" return;\n",
" } else {\n",
" this._key = event.key;\n",
" }\n",
" }\n",
" if (name === 'key_release') {\n",
" this._key = null;\n",
" }\n",
"\n",
" var value = '';\n",
" if (event.ctrlKey && event.key !== 'Control') {\n",
" value += 'ctrl+';\n",
" }\n",
" else if (event.altKey && event.key !== 'Alt') {\n",
" value += 'alt+';\n",
" }\n",
" else if (event.shiftKey && event.key !== 'Shift') {\n",
" value += 'shift+';\n",
" }\n",
"\n",
" value += 'k' + event.key;\n",
"\n",
" this._key_event_extra(event, name);\n",
"\n",
" this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n",
" return false;\n",
"};\n",
"\n",
"mpl.figure.prototype.toolbar_button_onclick = function (name) {\n",
" if (name === 'download') {\n",
" this.handle_save(this, null);\n",
" } else {\n",
" this.send_message('toolbar_button', { name: name });\n",
" }\n",
"};\n",
"\n",
"mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n",
" this.message.textContent = tooltip;\n",
"};\n",
"\n",
"///////////////// REMAINING CONTENT GENERATED BY embed_js.py /////////////////\n",
"// prettier-ignore\n",
"var _JSXTOOLS_RESIZE_OBSERVER=function(A){var t,i=new WeakMap,n=new WeakMap,a=new WeakMap,r=new WeakMap,o=new Set;function s(e){if(!(this instanceof s))throw new TypeError(\"Constructor requires 'new' operator\");i.set(this,e)}function h(){throw new TypeError(\"Function is not a constructor\")}function c(e,t,i,n){e=0 in arguments?Number(arguments[0]):0,t=1 in arguments?Number(arguments[1]):0,i=2 in arguments?Number(arguments[2]):0,n=3 in arguments?Number(arguments[3]):0,this.right=(this.x=this.left=e)+(this.width=i),this.bottom=(this.y=this.top=t)+(this.height=n),Object.freeze(this)}function d(){t=requestAnimationFrame(d);var s=new WeakMap,p=new Set;o.forEach((function(t){r.get(t).forEach((function(i){var r=t instanceof window.SVGElement,o=a.get(t),d=r?0:parseFloat(o.paddingTop),f=r?0:parseFloat(o.paddingRight),l=r?0:parseFloat(o.paddingBottom),u=r?0:parseFloat(o.paddingLeft),g=r?0:parseFloat(o.borderTopWidth),m=r?0:parseFloat(o.borderRightWidth),w=r?0:parseFloat(o.borderBottomWidth),b=u+f,F=d+l,v=(r?0:parseFloat(o.borderLeftWidth))+m,W=g+w,y=r?0:t.offsetHeight-W-t.clientHeight,E=r?0:t.offsetWidth-v-t.clientWidth,R=b+v,z=F+W,M=r?t.width:parseFloat(o.width)-R-E,O=r?t.height:parseFloat(o.height)-z-y;if(n.has(t)){var k=n.get(t);if(k[0]===M&&k[1]===O)return}n.set(t,[M,O]);var S=Object.create(h.prototype);S.target=t,S.contentRect=new c(u,d,M,O),s.has(i)||(s.set(i,[]),p.add(i)),s.get(i).push(S)}))})),p.forEach((function(e){i.get(e).call(e,s.get(e),e)}))}return s.prototype.observe=function(i){if(i instanceof window.Element){r.has(i)||(r.set(i,new Set),o.add(i),a.set(i,window.getComputedStyle(i)));var n=r.get(i);n.has(this)||n.add(this),cancelAnimationFrame(t),t=requestAnimationFrame(d)}},s.prototype.unobserve=function(i){if(i instanceof window.Element&&r.has(i)){var n=r.get(i);n.has(this)&&(n.delete(this),n.size||(r.delete(i),o.delete(i))),n.size||r.delete(i),o.size||cancelAnimationFrame(t)}},A.DOMRectReadOnly=c,A.ResizeObserver=s,A.ResizeObserverEntry=h,A}; // eslint-disable-line\n",
"mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
"\n",
"mpl.extensions = [\"eps\", \"jpeg\", \"pgf\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
"\n",
"mpl.default_extension = \"png\";/* global mpl */\n",
"\n",
"var comm_websocket_adapter = function (comm) {\n",
" // Create a \"websocket\"-like object which calls the given IPython comm\n",
" // object with the appropriate methods. Currently this is a non binary\n",
" // socket, so there is still some room for performance tuning.\n",
" var ws = {};\n",
"\n",
" ws.binaryType = comm.kernel.ws.binaryType;\n",
" ws.readyState = comm.kernel.ws.readyState;\n",
" function updateReadyState(_event) {\n",
" if (comm.kernel.ws) {\n",
" ws.readyState = comm.kernel.ws.readyState;\n",
" } else {\n",
" ws.readyState = 3; // Closed state.\n",
" }\n",
" }\n",
" comm.kernel.ws.addEventListener('open', updateReadyState);\n",
" comm.kernel.ws.addEventListener('close', updateReadyState);\n",
" comm.kernel.ws.addEventListener('error', updateReadyState);\n",
"\n",
" ws.close = function () {\n",
" comm.close();\n",
" };\n",
" ws.send = function (m) {\n",
" //console.log('sending', m);\n",
" comm.send(m);\n",
" };\n",
" // Register the callback with on_msg.\n",
" comm.on_msg(function (msg) {\n",
" //console.log('receiving', msg['content']['data'], msg);\n",
" var data = msg['content']['data'];\n",
" if (data['blob'] !== undefined) {\n",
" data = {\n",
" data: new Blob(msg['buffers'], { type: data['blob'] }),\n",
" };\n",
" }\n",
" // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
" ws.onmessage(data);\n",
" });\n",
" return ws;\n",
"};\n",
"\n",
"mpl.mpl_figure_comm = function (comm, msg) {\n",
" // This is the function which gets called when the mpl process\n",
" // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
"\n",
" var id = msg.content.data.id;\n",
" // Get hold of the div created by the display call when the Comm\n",
" // socket was opened in Python.\n",
" var element = document.getElementById(id);\n",
" var ws_proxy = comm_websocket_adapter(comm);\n",
"\n",
" function ondownload(figure, _format) {\n",
" window.open(figure.canvas.toDataURL());\n",
" }\n",
"\n",
" var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n",
"\n",
" // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
" // web socket which is closed, not our websocket->open comm proxy.\n",
" ws_proxy.onopen();\n",
"\n",
" fig.parent_element = element;\n",
" fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
" if (!fig.cell_info) {\n",
" console.error('Failed to find cell for figure', id, fig);\n",
" return;\n",
" }\n",
" fig.cell_info[0].output_area.element.on(\n",
" 'cleared',\n",
" { fig: fig },\n",
" fig._remove_fig_handler\n",
" );\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_close = function (fig, msg) {\n",
" var width = fig.canvas.width / fig.ratio;\n",
" fig.cell_info[0].output_area.element.off(\n",
" 'cleared',\n",
" fig._remove_fig_handler\n",
" );\n",
" fig.resizeObserverInstance.unobserve(fig.canvas_div);\n",
"\n",
" // Update the output cell to use the data from the current canvas.\n",
" fig.push_to_output();\n",
" var dataURL = fig.canvas.toDataURL();\n",
" // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
" // the notebook keyboard shortcuts fail.\n",
" IPython.keyboard_manager.enable();\n",
" fig.parent_element.innerHTML =\n",
" '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
" fig.close_ws(fig, msg);\n",
"};\n",
"\n",
"mpl.figure.prototype.close_ws = function (fig, msg) {\n",
" fig.send_message('closing', msg);\n",
" // fig.ws.close()\n",
"};\n",
"\n",
"mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n",
" // Turn the data on the canvas into data in the output cell.\n",
" var width = this.canvas.width / this.ratio;\n",
" var dataURL = this.canvas.toDataURL();\n",
" this.cell_info[1]['text/html'] =\n",
" '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
"};\n",
"\n",
"mpl.figure.prototype.updated_canvas_event = function () {\n",
" // Tell IPython that the notebook contents must change.\n",
" IPython.notebook.set_dirty(true);\n",
" this.send_message('ack', {});\n",
" var fig = this;\n",
" // Wait a second, then push the new image to the DOM so\n",
" // that it is saved nicely (might be nice to debounce this).\n",
" setTimeout(function () {\n",
" fig.push_to_output();\n",
" }, 1000);\n",
"};\n",
"\n",
"mpl.figure.prototype._init_toolbar = function () {\n",
" var fig = this;\n",
"\n",
" var toolbar = document.createElement('div');\n",
" toolbar.classList = 'btn-toolbar';\n",
" this.root.appendChild(toolbar);\n",
"\n",
" function on_click_closure(name) {\n",
" return function (_event) {\n",
" return fig.toolbar_button_onclick(name);\n",
" };\n",
" }\n",
"\n",
" function on_mouseover_closure(tooltip) {\n",
" return function (event) {\n",
" if (!event.currentTarget.disabled) {\n",
" return fig.toolbar_button_onmouseover(tooltip);\n",
" }\n",
" };\n",
" }\n",
"\n",
" fig.buttons = {};\n",
" var buttonGroup = document.createElement('div');\n",
" buttonGroup.classList = 'btn-group';\n",
" var button;\n",
" for (var toolbar_ind in mpl.toolbar_items) {\n",
" var name = mpl.toolbar_items[toolbar_ind][0];\n",
" var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
" var image = mpl.toolbar_items[toolbar_ind][2];\n",
" var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
"\n",
" if (!name) {\n",
" /* Instead of a spacer, we start a new button group. */\n",
" if (buttonGroup.hasChildNodes()) {\n",
" toolbar.appendChild(buttonGroup);\n",
" }\n",
" buttonGroup = document.createElement('div');\n",
" buttonGroup.classList = 'btn-group';\n",
" continue;\n",
" }\n",
"\n",
" button = fig.buttons[name] = document.createElement('button');\n",
" button.classList = 'btn btn-default';\n",
" button.href = '#';\n",
" button.title = name;\n",
" button.innerHTML = '<i class=\"fa ' + image + ' fa-lg\"></i>';\n",
" button.addEventListener('click', on_click_closure(method_name));\n",
" button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
" buttonGroup.appendChild(button);\n",
" }\n",
"\n",
" if (buttonGroup.hasChildNodes()) {\n",
" toolbar.appendChild(buttonGroup);\n",
" }\n",
"\n",
" // Add the status bar.\n",
" var status_bar = document.createElement('span');\n",
" status_bar.classList = 'mpl-message pull-right';\n",
" toolbar.appendChild(status_bar);\n",
" this.message = status_bar;\n",
"\n",
" // Add the close button to the window.\n",
" var buttongrp = document.createElement('div');\n",
" buttongrp.classList = 'btn-group inline pull-right';\n",
" button = document.createElement('button');\n",
" button.classList = 'btn btn-mini btn-primary';\n",
" button.href = '#';\n",
" button.title = 'Stop Interaction';\n",
" button.innerHTML = '<i class=\"fa fa-power-off icon-remove icon-large\"></i>';\n",
" button.addEventListener('click', function (_evt) {\n",
" fig.handle_close(fig, {});\n",
" });\n",
" button.addEventListener(\n",
" 'mouseover',\n",
" on_mouseover_closure('Stop Interaction')\n",
" );\n",
" buttongrp.appendChild(button);\n",
" var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n",
" titlebar.insertBefore(buttongrp, titlebar.firstChild);\n",
"};\n",
"\n",
"mpl.figure.prototype._remove_fig_handler = function (event) {\n",
" var fig = event.data.fig;\n",
" if (event.target !== this) {\n",
" // Ignore bubbled events from children.\n",
" return;\n",
" }\n",
" fig.close_ws(fig, {});\n",
"};\n",
"\n",
"mpl.figure.prototype._root_extra_style = function (el) {\n",
" el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n",
"};\n",
"\n",
"mpl.figure.prototype._canvas_extra_style = function (el) {\n",
" // this is important to make the div 'focusable\n",
" el.setAttribute('tabindex', 0);\n",
" // reach out to IPython and tell the keyboard manager to turn it's self\n",
" // off when our div gets focus\n",
"\n",
" // location in version 3\n",
" if (IPython.notebook.keyboard_manager) {\n",
" IPython.notebook.keyboard_manager.register_events(el);\n",
" } else {\n",
" // location in version 2\n",
" IPython.keyboard_manager.register_events(el);\n",
" }\n",
"};\n",
"\n",
"mpl.figure.prototype._key_event_extra = function (event, _name) {\n",
" var manager = IPython.notebook.keyboard_manager;\n",
" if (!manager) {\n",
" manager = IPython.keyboard_manager;\n",
" }\n",
"\n",
" // Check for shift+enter\n",
" if (event.shiftKey && event.which === 13) {\n",
" this.canvas_div.blur();\n",
" // select the cell after this one\n",
" var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
" IPython.notebook.select(index + 1);\n",
" }\n",
"};\n",
"\n",
"mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
" fig.ondownload(fig, null);\n",
"};\n",
"\n",
"mpl.find_output_cell = function (html_output) {\n",
" // Return the cell and output element which can be found *uniquely* in the notebook.\n",
" // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
" // IPython event is triggered only after the cells have been serialised, which for\n",
" // our purposes (turning an active figure into a static one), is too late.\n",
" var cells = IPython.notebook.get_cells();\n",
" var ncells = cells.length;\n",
" for (var i = 0; i < ncells; i++) {\n",
" var cell = cells[i];\n",
" if (cell.cell_type === 'code') {\n",
" for (var j = 0; j < cell.output_area.outputs.length; j++) {\n",
" var data = cell.output_area.outputs[j];\n",
" if (data.data) {\n",
" // IPython >= 3 moved mimebundle to data attribute of output\n",
" data = data.data;\n",
" }\n",
" if (data['text/html'] === html_output) {\n",
" return [cell, data, j];\n",
" }\n",
" }\n",
" }\n",
" }\n",
"};\n",
"\n",
"// Register the function which deals with the matplotlib target/channel.\n",
"// The kernel may be null if the page has been refreshed.\n",
"if (IPython.notebook.kernel !== null) {\n",
" IPython.notebook.kernel.comm_manager.register_target(\n",
" 'matplotlib',\n",
" mpl.mpl_figure_comm\n",
" );\n",
"}\n"
],
"text/plain": [
"<IPython.core.display.Javascript object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAj0AAAGtCAYAAAD9H8XfAAAgAElEQVR4nO3dfXRU9Z3H8RtkYiDPkRiQIOAqBGs1muXBqgyIIj4hSKsWFUSpEqNdH6CssBB8RLQ96xIRBRHlYK0gpLAoqxRBVo0S0fqsHIUkukKihocCQxKSz/7B4XZmMpPJZG4mk/zer3O+5zT33slNrjO/eXcyCZYAAAAMYLX1FwAAABANRA8AADAC0QMAAIxA9AAAACMQPQAAwAhEDwAAMALRAwAAjED0AAAAIxA9AADACEQPAAAwAtEDAACMQPQAAAAjED0AAMAIRA8AADAC0QMAAIxA9AAAACMQPQAAwAhEDwAAMALRAwAAjED0AAAAIxA9AADACEQPAAAwAtEDAACMQPQAAAAjED0AAMAIRA8AADAC0QMAAIxA9AAAACMQPQAAwAhEDwAAMALRA4ShsLBQWVlZAfdde+21crvd9sdLly6VZVnyeDzN+twbNmzQf/7nfzrxZXYYP/zwg0aNGqXk5GRZlqWPPvoo4HG9e/eWZVmyLEsul0unnnqq/vCHP+jAgQONjm1oaNDixYuVl5enLl26KDU1VSNHjtTmzZsbHet2u3Xttdc6/n0F0rt3b02fPj0q5wJMRfQAYQgneqqqqlRSUqKGhoZmfe7p06erd+/eDnyVHUdBQYGys7O1fv16lZSU6ODBgwGP6927t2688UaVlJTorbfe0gMPPCCXy6Vbbrml0bGTJ0+Wy+XSvffeq7/97W9au3atrr76anXq1EnLli3zOZboAToWogcIQzjRE662iJ5Dhw5F9XzhGjFihH7zm9+EPC5QMNx2221KSEhQfX29ve2VV16RZVlaunRpo88xYcIEde3aVd999529jegBOhaiBwhDJD/eqqmp0V133aWePXsqPj5ePXv21Pjx4+3Pe+zHM8dm4sSJ9ud6+eWXdcYZZyg+Pl69evVSYWGhjhw54nP+DRs26PTTT1dCQoKGDBmijz/+WJZlaeHChfYxvXv31h/+8AfNmTNHJ510kpKTkyVJ//u//6vLLrtMJ554opKSkjRw4ECtX78+4Pf+zjvv6Oyzz1ZCQoIuuOAClZWVaffu3bryyiuVmJioAQMGBPxRkb8dO3boqquuUlJSkpKSknTVVVdpx44d9n7/69FUEAYKhqeeekqWZWn37t32Nrfbrf79+/uE0DFlZWXq3Lmz5syZ43N8U9Ezc+ZMnXTSSY1ezVu+fLk6d+6sH3/8UZI0b948nX322UpKSlKPHj107bXXateuXU1+D4G+p4ULF8qyfJftqqoq3XTTTerWrZsSEhJ04YUX6osvvvA55sEHH1Tfvn0VHx+vrKwsXXnlldq/f3/Q7wvoqIgeIAzHnvjr6uoazTXXXNNk9MyZM0fdu3fX888/r82bN+vPf/6zHTbfffedbrzxRnXv3l0lJSUqKSnRN998I0l6/fXXZVmWJk2apP/5n//Rww8/rM6dO2vGjBn2uSoqKpSQkKBRo0bp1Vdf1YIFC5STkxMwerp3766RI0fqv//7v/XXv/5V0tEn6ccff1zr16/Xhg0bdM8996hTp0565513fL73xMREnXHGGVq+fLmKi4vVq1cvud1uud1uPf7443r99dc1YsQIdevWrcn3Mh0+fFh9+/ZVTk6OVqxYoRUrVqhfv37q27evampqJEklJSU6++yzddFFF6mkpEQffvhh0M8XKBD+/d//XcnJyXYc1tbWKj4+Xvfcc0/Qz3POOefowgsvtD8OFT2ffPKJLMvSli1bfLaPHj1aI0eOtD/+t3/7Ny1btkybNm3SK6+8oiFDhuj000/3ia+WRI/H49EZZ5yh0047TcuXL9e6des0bNgwnXTSSfaPAp9//nklJSXpqaee0ltvvaWVK1dq8uTJ+umnn4J+X0BHRfQAYQj0ioz3NBU9l19+eZNPuMF+vDV48GBddNFFPtvuv/9+de3aVT///LMk6d5771VWVpYdDJK0YMGCgNHTs2dPn+P81dfXq66uTiNHjtTkyZMbfe/eT/DHzvHggw/a2z7//HNZlqU33ngj6DkWLlyozp07q6yszN727bff6rjjjtPTTz9tb2vuj5eOvYJVV1engwcPav369UpLS9Ojjz5qH7Nr1y5ZlqUnnngi6OcZM2aMcnJywjr/gAEDdMcdd9gf79u3T8cff7yeffbZgMcfOXJE33//vSzL0ttvv+3zPYQbPYsWLdLxxx+vnTt32tv279+v9PR0FRUVSTr6vqirr766ye8BMAXRA4ShsLBQJ5xwgkpLSxvNyJEjm4yemTNnKiMjQ48//rg+++yzRp87UPQcOXJELpdLzzzzjM/2r776yidAhg4dqhtuuMHnmO+++y5g9Hj/2OyYn376SVOmTFF2drY6deoUMOIKCwuVkJDg86OcY69ClZSU2NtqamqCvm/mmEmTJuncc89ttH3IkCG6+eab7Y/DiR7/AB0zZozPMa0VPcdewTv2qs2yZcvkcrnsIJWkLVu2aNiwYUpLS/P5Gr2vUUui57rrrtP555/f6FXHkSNH6qabbpIkLV68WAkJCSosLFRpaWnAH+0BpiB6gDBE8p4ej8ejmTNnKjs7236PynPPPWcfHyh6du/eLcuy7B9DHbNnzx5ZlqW//OUvkqT+/fvr7rvv9jnm8OHDQd/T4++yyy5Tjx49tHDhQm3atEmlpaW69NJLNXjw4Ca/902bNsmyLH355Zc+2/3P62/UqFGNokSSrrrqKo0aNcr+OJzomThxokpLS7V582ZNnDhRlmVpwYIF9jHN/fHWiBEjwjr/l19+KcuytGnTJknSFVdcocsuu8zev3PnTiUmJmrUqFFavXq13n33Xb333nsB/9uEGz0XXXRR0Fcdj30f9fX1+uMf/6h+/frJsixlZWVp7ty5TX5PQEdF9ABhcOrv9HzxxRe69dZbZVmWtm3bJqnpV3oWLVrksz2SV3r8n0gPHTqkTp06NfpxzLBhw1oteiZNmqRf/epXjbZH8kqP//c1fPhwdevWzedv9bjdbg0YMCDgnxEoLy8P+43Mx5x55pnKz8/X3r17FR8fr+eff97e98wzz8jlcvn8plxZWVnI/zaBQvaRRx7xiZ5rrrlGQ4YMCfjK41dffdXo69y5c6f+4z/+Q5ZlafXq1SG/L6CjIXqAMDj5xwn3798vy7L04osvSpJmzZoV8HMPGTJEF198sc+2Bx54QF27dlV1dbWk8N7T4x8Hx1418v4bNRUVFYqPj2+16Hn66afVuXNnlZeX29t27NgR0Xt6/L+v999/X5Zl6b/+67/sbcd+Zf2FF15o9DkmTZrU4l9Zf/jhh3XiiSdqyZIlio+P1969e+19TzzxhBISElRbW2tve/TRR0P+txkxYoTPq17S0RD1jp6nn35aJ5xwgs+P0pojIyNDDz/8cFi3AToCogcIQyTRM2bMGD300EN69dVXtWHDBt1www3q0qWL/WvaL774ov0+j9LSUvvNqcfeNzN58mS9/vrrmjt3rlwuV8jf3urfv78sy/J5P1CwvwWTl5enU089VcXFxVqxYoUGDBigPn36tFr0HPvtrdNPP10rV67UihUrlJOT4/PbW1Jk0SNJF198sfr06ePz6/233HKLXC6Xpk6dqo0bN2rdunX69a9/rU6dOjWKoeae/5tvvpFlWerRo4dGjx7ts+/jjz9Wp06dNGHCBP3tb3/TI488Yv+3aSp6nnzySXXq1EmPPvqoXn/9dd1yyy3q1atXo9/eOuuss/TLX/7S/q3AFStW6I477tCf//xnSdKtt96q++67T2vWrNGbb74Z8DfzAFMQPUAYIomexx57TOecc46Sk5OVkpKi888/X2+++aZ9/OHDh3XzzTcrMzOz0d/p+ctf/qJf/OIXcrlcys7O1uzZsxv9nZ433nhDp59+uuLj45WXl6c333xTlmXppZdeso8JFgdfffWVhg4dqi5duuiUU07RkiVLNHHixFaLHunob2uNHj3a/js9o0eP9vk7PVLk0fPWW2/Jsiw7AKR//jMU55xzjv3PUFx88cX2e3Jacn7paDhalqXly5c32vfcc8+pT58+6tKli4YPH27/eLKp6KmtrdXdd9+tE088Uenp6br77rtVVFTU6O/0VFdXKz8/Xz169FB8fLxOPvlkXX/99fab5ZcuXapzzz1XaWlpSkxM1DnnnKOVK1c263sCOhqiB+ig1qxZI8uyGv2hOgAwFdEDdBDTpk3Tyy+/rE2bNqmoqEiZmZkaPnx4W39ZABAziB6ggygoKFDPnj3lcrnUrVs33XTTTWG/wRUAOjKiBwAAGIHoAQAARiB6AACAEYgeAABgBKIHAAAYgegJwbIsxcXFMQzDMEyHHv8/fNkRdfzvMEJxcXFt/SUAANDqTHi+I3pCMOFOAACACc93RE8IJtwJAAAw4fmO6AnBhDsBAAAmPN8RPSGYcCcAAMCE5zuiJwQT7gQAAJjwfEf0hGDCnQAAABOe74ieEEy4EwAAYMLzHdETggl3AgAATHi+I3pCMOFOAACACc93RE8IJtwJAAAw4fmO6AnBhDsBAAAmPN8RPSGYcCcAAMCE5zuiJwQT7gQAAJjwfEf0hNAad4Le09ep9/R1jn9eAABaiugB0QMAMALRA6IHAGAEogdEDwDACEQPiB4AgBGIHhA9AAAjED0dQH19vSZNmqShQ4fqyiuv1M8//xzW7YkeAIAJiJ4OYNWqVbr77rslSWvWrNH06dPDuj3RAwAwAdHTATz66KNavny5JOm7777TkCFDwro90QMAMAHRE0OKioqUl5en+Ph4jRs3zmdfbW2t8vPzlZaWpoyMDE2bNk0NDQ2SpFdffVUTJkyQJD333HPq379/WOclegAAJiB6YsiqVatUXFysgoKCRtEze/ZsDRw4UJWVlSovL1e/fv00f/58e/+MGTPkdrs1Y8YMXXjhhWGdl+gBAJiA6IlBhYWFjaInOztba9assT9etGiRcnNzG912+fLlPjHUHEQPAMAERE8M8o+e6upqWZalsrIye9vWrVvlcrnU0NCgqqoqud1ujRgxQnfeeadqa2ub/Pxz5sxRXFycPZbl/CUiegAAsYboiUH+0VNRUSHLsrRnzx572/bt22VZljweT8Tn45UeAIAJiJ4YFOyVnvLycntbaWmp/UpPpIgeAIAJiJ4YFOw9PWvXrrU/Xrx4ccD39LQE0QMAMAHRE0Pq6urk8Xg0c+ZMjR07Vh6PRzU1NZKkWbNmafDgwaqqqlJFRYVycnLCfsNyMEQPAMAERE8MKSwslGVZPuN2uyUd/Ts9U6ZMUWpqqtLT0zV16lRHfrQlET0AADMQPSB6AABGIHpA9AAAjED0gOgBABiB6AHRAwAwAtEDogcAYASiB0QPAMAIRA+IHgCAEYgeED0AACMQPSB6AABGIHpA9AAAjED0gOgBABiB6AHRAwAwAtEDogcAYASiB0QPAMAIRA+IHgCAEYgeED0AACMQPSB6AABGIHpA9AAAjED0gOgBABiB6AHRAwAwAtEDogcAYASiB0QPAMAIRA+IHgCAEYgeED0AACMQPSB6AABGIHpA9AAAjED0oFWjh/ABAMQKogdEDwDACEQPiB4AgBGIHhA9AAAjED0gegAARiB6QPQAAIxA9IDoAQAYgegB0QMAMALRA6IHAGAEogdEDwDACEQPiB4AgBGIHhA9AAAjED0gegAARiB6QPQAAIxA9IDoAQAYgegB0QMAMALRA6IHAGAEogdEDwDACEQPiB4AgBGIHhA9AAAjED0gegAARiB6QPQAAIxA9IDoAQAYgegB0QMAMALRA6IHAGAEogdEDwDACERPB/D555/L7XbL7XYrLy9PZ599dli3J3oAACYgejqYJ598Ug899FBYtyF6AAAmIHo6mPPOO0/ffvttWLchegAAJiB6YkhRUZHy8vIUHx+vcePG+eyrra1Vfn6+0tLSlJGRoWnTpqmhocHnmJ07d2rIkCFhn5foAQCYgOiJIatWrVJxcbEKCgoaRc/s2bM1cOBAVVZWqry8XP369dP8+fN9jpk7d66eeOKJsM9L9AAATED0xKDCwsJG0ZOdna01a9bYHy9atEi5ubk+x+Tm5mrXrl1hn4/oAQCYgOiJQf7RU11dLcuyVFZWZm/bunWrXC6X/SOuzz77TCNGjGjW558zZ47i4uLssSznLxHRAwCINURPDPKPnoqKClmWpT179tjbtm/fLsuy5PF4Ij4fr/QAAExA9MSgYK/0lJeX29tKS0t9XumJBNEDADAB0RODgr2nZ+3atfbHixcvbvSenpYiegAAJiB6YkhdXZ08Ho9mzpypsWPHyuPxqKamRpI0a9YsDR48WFVVVaqoqFBOTk6j395qKaIHAGACoieGFBYWyrIsn3G73ZKO/p2eKVOmKDU1Venp6Zo6daojP9qSiB4AgBmIHhA9AAAjED0gegAARiB60OrRQ/wAAGIB0QOiBwBgBKIHRA8AwAhED4geAIARiB4QPQAAIxA9IHoAAEYgekD0AACMQPSA6AEAGIHoAdEDADAC0QOiBwBgBKIHRA8AwAhED4geAIARiB4QPQAAIxA9IHoAAEYgekD0AACMQPSA6AEAGIHoAdEDADAC0YOoRQ/xAwBoS0QPiB4AgBGIHhA9AAAjED0gegAARiB6QPQAAIxA9IDoAQAYgehB1KOH8AEAtAWiB0QPAMAIRA+IHgCAEYgeED0AACMQPWiT6CF+AADRRvSA6AEAGIHoAdEDADAC0YM2jR7CBwAQLUQPiB4AgBGIHoe89dZb2rlzpyTphx9+0IQJEzRp0iRVVlZG4/QRIXoAACYgehwyYMAAlZWVSZJ++9vfauzYsRo/frzGjBkTjdNHpK2jh/gBAEQD0eOQlJQUSVJdXZ3S09O1d+9eeTwenXDCCdE4fUSIHgCACYgeh2RlZamyslIbN27UoEGDJEm1tbV2DMWyWIkewgcA0JqIHodMnTpVvXr10oknnqinnnpKklRSUqKzzjorGqePCNEDADAB0eOg119/XZs2bbI/Li0t1caNG6N1+hYjegAAJiB6HHbo0CFt3749mqeMWCxFD/EDAGgtRI9DfvzxR1111VWKi4tT165dJUmrV6/WvffeG43TR4ToAQCYgOhxyDXXXKMpU6aoqqpKaWlpkqSqqir9y7/8SzROH5FYjB7CBwDgNKLHIZmZmaqpqZEkpaen29v57S2iBwAQG4geh/Tt21c//vijpH9Gz//93//p1FNPjcbpIxKr0UP8AACcRPQ45L777tMll1yiL774Qunp6fr22281evRo3X///dE4fUSIHgCACYgeh9TW1uqee+5RYmKi4uLilJiYqHvuucf+kVcsi/XoIXwAAE4gelpBVVWVGhoaon3aFiN6AAAmIHocsmrVKn322Wc+2z799FOtXr06GqePSHuIHuIHABAposchp5xyinbv3u2zbdeuXfzKOtEDAIgRRI9DAv1qekNDg5KTk6Nx+oi0p+ghfgAALUX0OOTMM8/U22+/7bPt7bff1i9+8YtonD4iRA8AwAREj0OWLVumrKwszZs3T6tWrdK8efPUvXt3LVu2LBqnV9euXeV2u+V2u7V+/fqwbtseo4fwAQCEi+hxUHFxsUaNGqXTTz9dl156qf76179G69QRvaJE9AAATED0dBApKSm64IILNH78eP30009h3ba9Rg/xAwAIB9HjoE8++UQvvPCCFixY4DPNVVRUpLy8PMXHx2vcuHE++2pra5Wfn6+0tDRlZGRo2rRpPn8L6Ng/gbFkyRLddtttYX3dRA8AwAREj0MeeOABHX/88Ro0aJCGDRtmz/Dhw5v9OVatWqXi4mIVFBQ0ip7Zs2dr4MCBqqysVHl5ufr166f58+c3+hwHDx7UoEGDwvra23v0ED4AgOYgehySmZmpjz76yJHPVVhY2Ch6srOztWbNGvvjRYsWKTc3V5J04MABHTlyRJL02muv6frrrw/rfEQPAMAERI9DevbsqdraWkc+l3/0VFdXy7IslZWV2du2bt0ql8ulhoYGffDBB8rNzdXQoUN18cUXq7y8PKzzdYToIX4AAKEQPQ5ZuHChpk6d6kj4+EdPRUWFLMvSnj177G3bt2+XZVnyeDxhf/45c+YoLi7OHsty/hK1VfQQPgCAYIgeh3Tr1k2dO3eWy+VSZmamz4Qr2Cs93q/glJaW2q/0RKojvdJD9AAAgiF6HLJ58+agE65g7+lZu3at/fHixYvt9/REqqNFD/EDAAiE6IkhdXV18ng8mjlzpsaOHSuPx6OamhpJ0qxZszR48GBVVVWpoqJCOTk5AX97qyWIHgCACYgeh9TX1+uJJ57QoEGD1LdvX0nSxo0b9cILLzT7cxQWFsqyLJ9xu92Sjv6dnilTpig1NVXp6emaOnWqIz/akjpu9BA+AABvRI9DZsyYocGDB2vlypVKTU2VJO3YscOxH0G1po4cPcQPAOAYoschvXr1UlVVlSQpPT1dktTQ0KC0tLRonD4iRA8AwAREj0OysrLs998ci54DBw6oZ8+e0Th9REyIHuIHAED0OOS6667TjBkzJP0zegoLCzVhwoRonD4iRA8AwAREj0N2796tQYMGKSsrS507d1avXr00aNAgVVZWRuP0ETEpeggfADAX0eOAI0eOaOPGjTp06JC2bt2qFStW6L333lN9fX1rn9oRpkUP8QMAZiJ6HJKUlBSN07QKogcAYAKixyEXXnihY//KerSZGj2EDwCYhehxyPTp09WjRw9NmzZNTz75pBYsWGBPrDM5eogfADAH0eOQYcOGBZzhw4dH4/QRIXqIHgAwAdEDoofwAQAjED0OOXjwYNCJdUQP8QMAJiB6HBIXF6dOnToFnFhH9BA9AGACoschZWVlPvPuu+/qiiuu0JIlS6Jx+ogQPYQPAJiA6GlF1dXV6t+/f1udvtmIHuIHAExA9LSiyspKpaamttXpm43oIXoAwAREj0OmTZvmMwUFBTrllFN0ww03ROP0ESF6CB8AMAHR45CbbrrJZ+644w4tWbJENTU10Th9RIge4gcATED0gOghfgDACESPQxYsWKAPPvjAZ9sHH3ygp556KhqnjwjRQ/QAgAmIHodkZ2drz549Ptv27NmjXr16ReP0ESF6CB8AMAHR45DU1FTV19f7bKuvr1dKSko0Th8Roof4AQATED0OGTRokNat830SfPXVV5WXlxeN00eE6CF8AMAERI9D1q9fr6SkJOXn5+tPf/qT8vPzlZKSovXr10fj9BEheogeADAB0eOgDz74QFOmTNFll12m/Px8ffjhh9E6dUSIHuIHAExA9IDoIXwAwAhEj0NuuukmbdmyxWfbli1bdPPNN0fj9BEheogfADAB0eOQbt26qba21mdbTU2NMjMzo3H6iBA9RA8AmIDocUhmZqY8Ho/PNo/Ho4yMjGicPiJED+EDACYgehxyxRVX6MEHH/TZ9tBDD+nSSy+NxukjQvQQPwBgAqLHIV9//bV69eqlAQMG6Morr9SAAQN08skna/v27dE4fUSIHsIHAExA9DjoH//4h1588UU98MADWrFihQ4cOBCtU0eE6CF6AMAERI8Dtm3bpuHDh8vlcqlTp05yuVwaNmyYSktLW/vUjiB6iCAAMAHRE6Ft27YpOTlZkydP1saNG/Xll19q48aNuuWWW5ScnKxt27a15ukdQfQQPQBgAqInQpdeeqnmzZsXcN+8efN0ySWXtObpHUH0ED4AYAKiJ0Kpqanau3dvwH179+7lX1lniB8AiBFET4SSkpJ08ODBgPsOHDigpKSk1jy9I4ieth8AQOsjeiJ0wQUX6Lnnngu477nnntP555/fmqd3BNETOwMAaD1ET4Q2bNig5ORkzZ07Vzt37lRNTY127typRx55RMnJyXrjjTda8/SOIHpiawAArYPoccCaNWvUt29fderUyZ6+ffuquLi4tU/tCKInNgcA4Cyix0Fff/213nnnHX399dfROqUjiJ7YHwBA5IgeED3tYAAAkSN6QPS0swEAtAzRA6KnnQ4AIDxED4iedjwAgOYjekD0dIABAIRG9IDo6SADAGga0QOipwMOAKAxogdETwccAEBjRA+Ing4+AICjiB4QPYYMAJiO6AHRY+AAgImIng5g9+7dOvfcczV06FCdd955+vTTT8O6PdFj3gCAiYieDuDIkSOqr6+XJG3atEnXX399WLcneswdADAJ0dPBFBcX67HHHgvrNkSP2QMApiB6YkhRUZHy8vIUHx+vcePG+eyrra1Vfn6+0tLSlJGRoWnTpqmhocHe//nnn+vcc89Vdna23n///bDOS/QwBBAAExA9MWTVqlUqLi5WQUFBo+iZPXu2Bg4cqMrKSpWXl6tfv36aP39+o8/x97//XYMGDQrrvEQPQ/wAMAHRE4MKCwsbRU92drbWrFljf7xo0SLl5uZKkg4fPmxvLysr07Bhw8I6H9HDED4ATED0xCD/6KmurpZlWSorK7O3bd26VS6XSw0NDXr33Xc1dOhQDRs2TG63W9u2bQvrfEQPQ/wAMAHRE4P8o6eiokKWZWnPnj32tu3bt8uyLHk8nrA//5w5cxQXF2ePZTl/idr6iZohgADAH9ETg4K90lNeXm5vKy0ttV/piRSv9DDEDwATED0xKNh7etauXWt/vHjxYvs9PZEiehgCCIAJiJ4YUldXJ4/Ho5kzZ2rs2LHyeDyqqamRJM2aNUuDBw9WVVWVKioqlJOTE/C3t1qC6GEIHwAmIHpiSGFhoSzL8hm32y3p6N/pmTJlilJTU5Wenq6pU6c68qMtiehhiB8AZiB6QPQwrTIAEGuIHhA9TKsMAMQaogdED9PqAwCxgOgB0cO0+gBALCB6QPQwbTIAEG1ED4gepk0GAKKN6AHRw7TJeN9PACAaiB4QPUxMDAC0NqIHRA8TUwMArYXoAdHDxNwAQGsgekD0MDE7AOAkogdEDxPTAwBOIXpA9JNNE/0AABJkSURBVDDtbgCgJYgeED1Muxvv+xkANBfRA6KH6RADAKEQPSB6mA43ABAI0QOih+mQAwD+iB4QPYwxA8BsRA+IHsaoAWAuogdED2PUBLqPAjAD0QOihzFqAt1HvbcB6LiIHhA9jPET6H4LoOMhekD0MMZPoPttoI8BtG9ED4gehgljALRfRA+IHoYJcwC0T0QPiB6GCXMC3c+DPQ4AxA6iB0QPwzg4/o+DQP8bQNsgekD0MIzD4/04CPSYANA2iB4QPQzTihPoMdGcxw8A5xE9IHoYphUn0GPCe1uwxw8A5xE9IHoYJsoT6HHi//gB4DyiB0QPw8TA+D9uADiP6AHRwzAxMP6PGwDOI3pA9DBMDE6gx5L3xwDCR/SA6GGYdjLejy0A4SN6QPQwTDsZ78cWgPARPSB6GKadjPdjy/uxBqB5iB4QPQzTzsf7cQcgOKIHRA/DtPPxftwBCI7oAdHDMO18vB93AIIjekD0MEw7n1CPOwBHET0gehimg4/3YxIwGdEDoodhOvh4Pyb9/zdgEqIHRA/DdPDxfkwGenwCpiB64PidoK0XeIZhfMf7cRnoMQqYgugB0cMwBk2gxyhgCqIHRA/DMD4DdFRED4gehmF8BuioiB4QPQzD+EygxzPQERA9IHoYhvGZQI9noCMgekD0MAzjM4Eez/6P7UCPeyDWET0gehiGCTjej2f/x3agxz0Q64ieDqC2tla/+tWvlJqaqpUrV4Z9e6KHYZhQ4//YDvS4B2Id0dMBNDQ06IcfflBhYSHRwzBMq4z/YzvQ4x2IdURPB0L0MAzTWuP/2A70ePfeBsQioieGFBUVKS8vT/Hx8Ro3bpzPvtraWuXn5ystLU0ZGRmaNm2aGhoafI4hehiGaa3xf2wHerx7bwNiEdETQ1atWqXi4mIVFBQ0ip7Zs2dr4MCBqqysVHl5ufr166f58+f7HEP0MAwTrQn0ePfeFs5aAUQL0RODCgsLG0VPdna21qxZY3+8aNEi5ebmNrod0cMwTDQm0OM92BoQaq0AooXoiUH+0VNdXS3LslRWVmZv27p1q1wul/0jrt/85jfq27evfvnLX2ratGlhnY/oYRgm3An0eA+2BoRaK4BoIXpikH/0VFRUyLIs7dmzx962fft2WZYlj8cT9uefM2eO4uLi7LEsZy9RWy/GDMO0/gR6vIdaA4KtFd4fA62J6IlBwV7pKS8vt7eVlpb6vNITCV7pYRgm3An0eA+1BjRnrQBaE9ETg4K9p2ft2rX2x4sXL270np6WInoYhgl3Aj3eQ60BzVkrgNZE9MSQuro6eTwezZw5U2PHjpXH41FNTY0kadasWRo8eLCqqqpUUVGhnJycRr+91VJED8Mw4U6gx3uoNaA5a0Wg/YBTiJ4YUlhYKMuyfMbtdks6+nd6pkyZotTUVKWnp2vq1KmO/GhLInoYhgl/Aj3eQ60BzVkrAu0HnEL0gOhhGCbsaenjvblrhf9+wAlED4gehmHCnpY+3pu7VvjvB5xA9IDoYRgm7In08R7qtv77AScQPSB6GIYJeyJ9vIe6rf9+wAlED4gehmFiboKtJU1tB0IhekD0MAwTcxNsLWlqOxAK0QOih2GYmJtga0lT24FQiB4QPQzDtJsJtsZ4rz+B1iNAInogoodhmPYzwdYY7/Un0HoESEQPRPQwDNN+Jtga473+BFqPAInogYgehmHazwRbY7zXn0DrESARPRDRwzBM+5lga4z3+hNqLYK5iB4QPQzDtJsJtsZ4rz+h1iKYi+gB0cMwTLuZYGuM9/oTai2CuYgeED0Mw7SbCbbGeK8/odYimIvoAdHDMEy7mWBrjPf6E2otgrmIHhA9DMO0mwm2xnivP6HWIpiL6AHRwzBMux/v9acla5H38c1Z35qz/jVnrUR0ET0gehiGaffjvf60ZC3yPr4561tz1r/mrJWILqIHRA/DMO1+vNeflqxF3sc3Z31rzvrXnLUS0UX0gOhhGKbdj/f605K1yPv45qxvzVn/mrNWIrqIHhA9DMO0+/Fef1qyFnkf35z1rTnrX3PWSkQX0QOih2GYdj/e609L1iLv45uzvjVn/WvOWonoInpA9DAM0+7He/1pyVrkfXxz1rfmrH/NWSsRXUQPiB6GYdr9eK8/LVmLvI9vzvrWnPWvOWsloovoAdHDMEy7H+/1pyVrkffxzVnfmrP+NWetRHQRPSB6GIZp9+O9/rRkLfI+vjnrW3PWv+aslYguogdED8MwjNeEu46FWv/8twc61ok1N9zbRLLfqdsFO96J6xII0QOih2EYxmvCXcdCrX/+2wMd68SaG+5tItnv1O2CHe/EdQmE6AHRwzAM4zXhrmOh1j//7YGOdWLNDfc2kex36nbBjnfiugRC9IDoYRiG8Zpw17FQ65//9kDHOrHmhnubSPY7dbtgxztxXQIhekD0MAzDeE2461io9c9/e6BjnVhzw71NJPudul2w4524LoEQPSB6GIZhvCbcdSzU+ue/PdCxTqy54d4mkv1O3S7Y8U5cl0CIHhA9DMMwXhPuOhZq/fPfHuhYJ9bccG8TyX6nbhfseCeuSyBED4gehmEYrwl3HQu1/vlvD3SsE2tuuLeJZL9Ttwt2vBPXJRCiB0QPwzCM14S7joVa//y3BzrWiTU33NtEst+p2wU73onrEgjRA6KHYRjGa8Jdx0Ktf/7bAx3rxJob7m0i2e/U7YId78R1CYToAdHDMAzjNeGuY6HWP//tgY51Ys0N9zaR7HfqdsGOd+K6BEL0gOhhGIbxmnDXsVDrn//2QMc6seaGe5tI9jt1u2DHO3FdAiF6QPQwDMN4TbjrWKj1z397oGOdWHPDvU0k+526XbDjnbgugRA9IHoYhmG8Jtx1LNT657890LFOrLnh3iaS/U7dLtjxTlyXQIgeED0MwzBeE+46Fmr9898e6Fgn1txwbxPJfqduF+x4J65LIEQPZFmW4uLiHJ3W+JwmDNeNa8Y1i93hurX/a2ZZHT8JOv53GIPi4jp+TbcGrlv4uGbh45q1DNctfFyz6CN62gB39JbhuoWPaxY+rlnLcN3CxzWLPqKnDXBHbxmuW/i4ZuHjmrUM1y18XLPoI3rawJw5c9r6S2iXuG7h45qFj2vWMly38HHNoo/oAQAARiB6AACAEYgeAABgBKIHAAAYgeiJotraWuXn5ystLU0ZGRmaNm2aGhoa2vrLiikTJ06Uy+VSYmKiPV9//bW9f9++fbr22muVlJSk7t27649//GMbfrVtp6ioSHl5eYqPj9e4ceN89oW6Rt9//70uueQSde3aVb1799by5cuj+aW3maaumdvtVnx8vM/97vDhw/Z+U6/Z4cOHNXnyZPXp00dJSUnKycnR0qVL7f3c1xoLdc24r7UtoieKZs+erYEDB6qyslLl5eXq16+f5s+f39ZfVkyZOHGi7r333qD7J0yYoNGjR2vfvn365JNPlJmZqbVr10bxK4wNq1atUnFxsQoKCho9gYe6RkOHDlV+fr4OHTqkTZs2KSkpSR999FG0v4Woa+qaud1uFRUVBb2tqdfswIEDmjVrlr755hs1NDSopKREaWlp2rhxoyTua4GEumbc19oW0RNF2dnZWrNmjf3xokWLlJub24ZfUexpKnoOHjyo+Ph4ffzxx/a2GTNmaMyYMdH68mJOYWGhzxN4qGv0zTffqHPnzvr555/t/ePHj9ddd90VvS+6jflfM6npJyKuma+xY8fq/vvv574WhmPXTOK+1taIniiprq6WZVkqKyuzt23dulUul4sfcXmZOHGi0tPTlZ6erjPOOEMLFy6093344Yc67rjjVF9fb29bsWKFTj311Lb4UmOC/xN4qGu0evVq9enTx+dzPPbYY7roooui8wXHgGDRc8IJJygjI0N5eXlatWqVvY9r9k8ej0c9e/bUK6+8wn2tmbyvmcR9ra0RPVFSUVEhy7K0Z88ee9v27dtlWZY8Hk8bfmWxZdu2bfrxxx915MgRbdmyRVlZWVq2bJkkacuWLUpNTfU5/o033lBWVlZbfKkxwf8JPNQ1WrZsmc466yyf/YsWLdLgwYNb/4uNEYGip6SkRPv27VNtba3Wrl2rxMREbd68WRLX7JiGhgZdf/31GjZsmOrr67mvNYP/NZO4r7U1oidKjr3SU15ebm8rLS3llZ4QHnnkEV1++eWS/vkqhvf1WrlyJa/0BHilJ9g1Wr16tfr27evzOR5//HGj/p9koOjxd+utt6qgoEAS10w6+uR922236V//9V+1d+9eSdzXQgl0zQLhvhZdRE8UZWdn+7zJb/HixbynJ4RHH31Ul112maR/vl/lk08+sffPnDmT9/QEeE9PsGsU6D0D119/vVHvGWhO9EyZMkW33367JK5ZQ0OD8vPzdfbZZ6u6utrezn0tuGDXLBDua9FF9ETRrFmzNHjwYFVVVamiokI5OTn89pafl19+Wfv371dDQ4Peffddde/eXc8++6y9/8Ybb9RVV12l/fv369NPP1VWVpaRv71VV1cnj8ejmTNnauzYsfJ4PKqpqZEU+hpdcMEFuv3223Xo0CG99dZbxvx2SLBrtmfPHr322ms6dOiQjhw5otdee02JiYnasGGDfVtTr5kk3X777TrzzDP1008/NdrHfS2wYNeM+1rbI3qiqLa2VlOmTFFqaqrS09M1depUfrTl54ILLlBqaqqSkpI0YMCARlG4b98+XXPNNUpKSlJWVpaxf6ensLBQlmX5jNvtlhT6Gn3//fcaOXKkunTpopNPPtmYvwMS7JpVVVVp4MCBSk5OVkpKinJzc/XSSy/53NbUa1ZWVibLsnT88cf7/F2Z2267TRL3tUCaumbc19oe0QMAAIxA9AAAACMQPQAAwAhEDwAAMALRAwAAjED0AAAAIxA9AADACEQPAAAwAtEDoN2zLEuffvppW38ZAGIc0QOgSW63W/Hx8UpMTFRGRoYuueQSff311z7H7N+/X3fddZeys7OVkJCgU045RXPmzFFtba19zNKlS5WXl9cqXyPRA6A5iB4ATXK73SoqKpJ09B+ZvOGGG3TeeefZ+2tra3Xuuedq+PDh+uqrr3TkyBFt27ZNZ511ln7961/bxxE9ANoa0QOgSd7RI0nr1q1T165d7Y+XLl2qE044Qfv27fO53Y4dO+RyubRp0yb7uGDRc+edd+rWW2/12Xbfffdp/PjxkqT169frnHPOUUpKinr06KG7775bdXV19rHe0TNx4kTde++99r6dO3fKsiz94x//kHT034u67bbb1LNnT2VlZel3v/udDhw4EO5lAdAOET0AmuQdPfv379dvf/tb5ebm2vuvu+463XjjjQFve/755+u+++6T1HT0bN26Venp6fa/FN/Q0KCTTz5Z69evlyRt3rxZf//731VfX6+vvvpKffv21ZNPPmnfPpzoufrqq3XjjTdq79692rdvn6644gr9/ve/b9G1AdC+ED0AmuR2u9WlSxelpKTIsiyddtpp+uyzz+z9F110kaZPnx7wttdcc40mT54sKfSPt/r376/Vq1dLkjZt2qSsrCwdOXIk4LEPPvigz4/Omhs9lZWVOu6443xeldq6dasyMzNDXQYAHQDRA6BJ3q/0fPPNNzrttNPsOJGceaVHOhoyV199tSTplltu0V133WXve++99zR8+HBlZmYqJSVFXbp00bBhw+z9zY2e999/X3FxcUpNTbUnJSVFCQkJ8ng84V4aAO0M0QOgSf7v6XnttdfUo0cPHTp0SJK0ZMmSJt/T8+abb0oKHT07duxQQkKCdu3apdTUVH3wwQf2vlNOOUVz587VwYMHJR0NJLfbbe/3jp6CggLl5+fb+9599107enbt2qXOnTsTOIChiB4ATfKPHknKzc3Vn/70J0lSTU2NBg0a5PPbWx9++KHOOussjR071r5Nc3576/zzz9fIkSM1YMAAn+2ZmZl6+umnJUmffPKJevfuHTR6nn32WfXp00e7d+9WdXW1Lr/8cp/39IwZM0a/+93v9PPPP6uhoUHff/+91q1b17KLA6BdIXoANClQ9Lz00kvKysqyX3nZt2+ffv/736tnz546/vjj1adPH82ePTvsv9PzzDPPyLIsPfzwwz7bV69erd69eysxMVEjRozQ9OnTg0bP4cOHNX78eKWkpOi0007T4sWLG/321p133qmTTz5ZycnJ6t+/v+bNm9fi6wOg/SB6AACAEYgeAABgBKIHAAAYgegBAABGIHoAAIARiB4AAGAEogcAABiB6AEAAEYgegAAgBGIHgAAYASiBwAAGOH/AaqzdBPrOxWmAAAAAElFTkSuQmCC\" width=\"639.85\">"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"Text(0.5, 0, 'ROI value')"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"print(f\"Dataset of shape {ds.shape} with dtype: {ds.dtype}, total size of {ds.nbytes/M:.3f} MB\")\n",
"print(f\"Read time (from NFS): {dt:.3f}s, Read speed: {ds.nbytes/M/dt:.3f} MB/s\")\n",
"m = ds.max()\n",
"print(\"This is a very compressible dataset with a maximum value of\", m)\n",
"fig, ax = subplots()\n",
"ax.hist(ds.ravel(), m+1)\n",
"ax.set_title(\"Histogram of ROI values\")\n",
"ax.set_yscale(\"log\")\n",
"ax.set_ylabel(\"Occurences\")\n",
"ax.set_xlabel(\"ROI value\")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "formed-ladder",
"metadata": {},
"outputs": [],
"source": [
"def bench(key):\n",
" res = None\n",
" os.system(\"sync\")\n",
" with tempfile.TemporaryDirectory() as d:\n",
" if key.format == \"numpy\":\n",
" fname = os.path.join(d, \"numpy.npy\")\n",
" t0 = time.perf_counter()\n",
" numpy.save(fname, ds)\n",
" t1 = time.perf_counter()\n",
" ds1 = numpy.load(fname)\n",
" t2 = time.perf_counter()\n",
" elif key.format == \"hdf5\":\n",
" fname = os.path.join(d, \"file.h5\")\n",
" if key.compression == \"bitshuffle\":\n",
" cmp = hdf5plugin.Bitshuffle()\n",
" else:\n",
" cmp = {\"compression\":key.compression}\n",
" t0 = time.perf_counter()\n",
" with h5py.File(fname, \"w\") as h:\n",
" h.create_dataset(\"data\", data=ds, chunks=key.chunk, **cmp)\n",
" t1 = time.perf_counter()\n",
" with h5py.File(fname, \"r\") as h:\n",
" ds1 = h[\"data\"][()]\n",
" t2 = time.perf_counter()\n",
" assert abs(ds1-ds).max() == 0\n",
" res = Res(ds.nbytes/os.stat(fname).st_size, ds.nbytes/M/(t2-t1), ds.nbytes/M/(t1-t0))\n",
" return res\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "canadian-battlefield",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{Key(format='numpy', compression=None, chunk=None): Res(compression=0.999999987672584, read=4557.3690840703375, write=1591.1733778477628)}\n"
]
}
],
"source": [
"results = {}\n",
"key = Key(\"numpy\", None, None)\n",
"results[key] = bench(key)\n",
"print(results)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "commercial-publisher",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Key(format='hdf5', compression=None, chunk=None) Res(compression=0.9999998027613801, read=4740.710243687177, write=1590.85635712532)\n",
"Key(format='hdf5', compression=None, chunk=(1, 1664)) Res(compression=0.9930296992060388, read=1217.2936255579752, write=625.5655888179406)\n",
"Key(format='hdf5', compression=None, chunk=(1, 6656)) Res(compression=0.9982481664538824, read=2576.1340075650933, write=987.3396274666414)\n",
"Key(format='hdf5', compression=None, chunk=(40, 6656)) Res(compression=0.9999557773502066, read=4584.855723496288, write=1400.5383090330129)\n",
"Key(format='hdf5', compression=None, chunk=(160, 1664)) Res(compression=0.9997507093506732, read=3825.364627557245, write=1374.6310640228137)\n"
]
}
],
"source": [
"for k in [(\"hdf5\", None, None),\n",
" (\"hdf5\", None, (1,1664)),\n",
" (\"hdf5\", None, (1,6656)),\n",
" (\"hdf5\", None, (40,6656)),\n",
" (\"hdf5\", None, (160,1664))]:\n",
" key = Key(*k)\n",
" results[key] = bench(key)\n",
" print(key, results[key])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "vocal-genome",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Key(format='hdf5', compression='gzip', chunk=None) Res(compression=39.78027719788861, read=477.12773076606135, write=189.29843477263105)\n",
"Key(format='hdf5', compression='gzip', chunk=(1, 1664)) Res(compression=26.602934295946124, read=317.32810699302485, write=170.03321179279254)\n",
"Key(format='hdf5', compression='gzip', chunk=(1, 6656)) Res(compression=34.53257668982696, read=446.63243669626235, write=230.92592586648894)\n",
"Key(format='hdf5', compression='gzip', chunk=(40, 6656)) Res(compression=38.81519874522585, read=530.0937871707857, write=187.2893327456712)\n",
"Key(format='hdf5', compression='gzip', chunk=(160, 1664)) Res(compression=39.37773055496157, read=518.6832108881214, write=187.79710544837886)\n"
]
}
],
"source": [
"for k in [(\"hdf5\", \"gzip\", None),\n",
" (\"hdf5\", \"gzip\", (1,1664)),\n",
" (\"hdf5\", \"gzip\", (1,6656)),\n",
" (\"hdf5\", \"gzip\", (40,6656)),\n",
" (\"hdf5\", \"gzip\", (160,1664))]:\n",
" key = Key(*k)\n",
" results[key] = bench(key)\n",
" print(key, results[key])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "conditional-colombia",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Key(format='hdf5', compression='bitshuffle', chunk=None) Res(compression=34.48097650060628, read=1175.538050913075, write=1132.9698839668529)\n",
"Key(format='hdf5', compression='bitshuffle', chunk=(1, 1664)) Res(compression=25.891379581397512, read=290.720847721375, write=226.63448569650572)\n",
"Key(format='hdf5', compression='bitshuffle', chunk=(1, 6656)) Res(compression=31.35532673245106, read=581.3902571212135, write=543.2917022423106)\n",
"Key(format='hdf5', compression='bitshuffle', chunk=(40, 6656)) Res(compression=34.34958110991905, read=2446.856758234593, write=1461.2653063086007)\n",
"Key(format='hdf5', compression='bitshuffle', chunk=(160, 1664)) Res(compression=34.36789580388631, read=1901.9301071433877, write=1569.7570068929156)\n"
]
}
],
"source": [
"for k in [(\"hdf5\", \"bitshuffle\", None),\n",
" (\"hdf5\", \"bitshuffle\", (1,1664)),\n",
" (\"hdf5\", \"bitshuffle\", (1,6656)),\n",
" (\"hdf5\", \"bitshuffle\", (40,6656)),\n",
" (\"hdf5\", \"bitshuffle\", (160,1664))]:\n",
" key = Key(*k)\n",
" results[key] = bench(key)\n",
" print(key, results[key])"
]
},
{
"cell_type": "code",
"execution_count": 47,
"id": "balanced-sheep",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Format | Compressor | chunks | Compression | Read (MB/s) | Write (MB/s)\n",
"numpy | None | None | 1.000 | 4557.369 | 1591.173\n",
"hdf5 | None | None | 1.000 | 4740.710 | 1590.856\n",
"hdf5 | None | (1, 1664) | 0.993 | 1217.294 | 625.566\n",
"hdf5 | None | (1, 6656) | 0.998 | 2576.134 | 987.340\n",
"hdf5 | None | (40, 6656) | 1.000 | 4584.856 | 1400.538\n",
"hdf5 | None | (160, 1664) | 1.000 | 3825.365 | 1374.631\n",
"hdf5 | gzip | None | 39.780 | 477.128 | 189.298\n",
"hdf5 | gzip | (1, 1664) | 26.603 | 317.328 | 170.033\n",
"hdf5 | gzip | (1, 6656) | 34.533 | 446.632 | 230.926\n",
"hdf5 | gzip | (40, 6656) | 38.815 | 530.094 | 187.289\n",
"hdf5 | gzip | (160, 1664) | 39.378 | 518.683 | 187.797\n",
"hdf5 | bitshuffle | None | 34.481 | 1175.538 | 1132.970\n",
"hdf5 | bitshuffle | (1, 1664) | 25.891 | 290.721 | 226.634\n",
"hdf5 | bitshuffle | (1, 6656) | 31.355 | 581.390 | 543.292\n",
"hdf5 | bitshuffle | (40, 6656) | 34.350 | 2446.857 | 1461.265\n",
"hdf5 | bitshuffle | (160, 1664) | 34.368 | 1901.930 | 1569.757\n"
]
}
],
"source": [
"print(f\"{'Format':10s} | {'Compressor':10s} | {'chunks':11s} | {'Compression':11s} | {'Read (MB/s)':11s} | {'Write (MB/s)':10s}\")\n",
"for k, v in results.items():\n",
" print(f\"{k.format:10s} | {str(k.compression):10s} | {str(k.chunk) or '-':11s} | {v.compression:11.3f} | {v.read:11.3f} | {v.write:10.3f}\")"
]
},
{
"cell_type": "markdown",
"id": "whole-thesaurus",
"metadata": {},
"source": [
"# Auto-chunking function\n",
"First, the default implementation:"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "correct-cambridge",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(1, 1664)"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def guess_chunk(scan_shape, detector_shape):\n",
" \"\"\"Simplified form of h5py._hl.filters.guess_chunk\"\"\"\n",
" if detector_shape:\n",
" return tuple(1 for _ in scan_shape) + tuple(\n",
" min(n, max(n // 4, 256)) if n else 256 for n in detector_shape)\n",
" else:\n",
" return tuple(min(n, 256) if n else 256 for n in scan_shape)\n",
"\n",
"guess_chunk((390000,), (6656,))"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "native-energy",
"metadata": {},
"outputs": [],
"source": [
"from math import ceil\n",
"def auto_chunk(scan_shape, detector_shape, \n",
" split=4, block=1<<20, dtype=numpy.int32):\n",
" \"\"\"Try to guess the optimal chunk size with those constrains:\n",
" * Always split any dimension for partial accesses\n",
" * close to the block size (approximative when compressing)\n",
" \n",
" :param scan_shape: tuple with the dimentions of the scan\n",
" :param detector_shape: tuple with the dimentions of every data-point\n",
" :param split: split the inner-most dimentions at least by this value\n",
" :param block: target size (in bytes) of chunk (uncompressed)\n",
" :param dtype: data type to get the size of every element\n",
" :return: tuple with the optimal chunk size\n",
" \"\"\"\n",
" if detector_shape:\n",
" shape = tuple(scan_shape) + tuple(detector_shape)\n",
" else:\n",
" shape = tuple(scan_shape)\n",
" itemsize = numpy.dtype(dtype).itemsize\n",
" size = numpy.prod(shape)\n",
" nbytes = size * itemsize\n",
" if nbytes > block:\n",
" target = block//itemsize\n",
" size = 1\n",
" lst = []\n",
" for i in shape[-1::-1]:\n",
" if size >=target:\n",
" j = 1\n",
" elif i * size / split > 2.0 * target:\n",
" j = int(ceil(target/size))\n",
" else:\n",
" j = int(ceil(i/split))\n",
" lst.insert(0, j)\n",
" size *= j\n",
" chunks = tuple(lst)\n",
" else:\n",
" chunks = shape\n",
" return chunks"
]
},
{
"cell_type": "code",
"execution_count": 46,
"id": "suburban-minnesota",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(158, 1664)\n",
"(5, 250, 250)\n",
"(1, 512, 512)\n",
"(10, 20)\n"
]
}
],
"source": [
"print(auto_chunk((390000,), (6656,)))\n",
"print(auto_chunk((100,), (1000, 1000)))\n",
"print(auto_chunk((100,), (2048, 2048)))\n",
"print(auto_chunk((10,), (20,)))"
]
},
{
"cell_type": "code",
"execution_count": 44,
"id": "digital-stock",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(40, 6656)\n",
"(1, 263, 1000)\n",
"(1, 128, 2048)\n"
]
}
],
"source": [
"print(auto_chunk((390000,), (6656,), split=1))\n",
"print(auto_chunk((100,), (1000, 1000), split=1))\n",
"print(auto_chunk((100,), (2048, 2048), split=1))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9 (local venv)",
"language": "python",
"name": "py39-venv"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment