Test of pre-trained normalisation approaches

Normalisation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "from fastai.script import *\n",
    "from fastai.vision import *\n",
    "from fastai.callbacks import *\n",
    "from fastai.distributed import *\n",
    "from fastprogress import fastprogress\n",
    "from torchvision.models import *\n",
    "from fastai.vision.models.xresnet import *\n",
    "from fastai.vision.models.xresnet2 import *\n",
    "from fastai.vision.models.presnet import *\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "torch.backends.cudnn.benchmark = True\n",
    "fastprogress.MAX_COLS = 80"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "## Stats collection code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "from typing import Iterable\n",
    "class RunningStatistics:\n",
    "    '''Records mean and variance of the final `n_dims` dimension over other dimensions across items. So collecting across `(l,m,n,o)` sized\n",
    "       items with `n_dims=1` will collect `(l,m,n)` sized statistics while with `n_dims=2` the collected statistics will be of size `(l,m)`.\n",
    "\n",
    "       Uses the algorithm from Chan, Golub, and LeVeque in \"Algorithms for computing the sample variance: analysis and recommendations\":\n",
    "\n",
    "       `variance = variance1 + variance2 + n/(m*(m+n)) * pow(((m/n)*t1 - t2), 2)`\n",
    "\n",
    "       This combines the variance for 2 blocks: block 1 having `n` elements with `variance1` and a sum of `t1` and block 2 having `m` elements\n",
    "       with `variance2` and a sum of `t2`. The algorithm is proven to be numerically stable but there is a reasonable loss of accuracy (~0.1% error).\n",
    "\n",
    "       Note that collecting minimum and maximum values is reasonably innefficient, adding about 80% to the running time, and hence is disabled by default.\n",
    "    '''\n",
    "    def __init__(self, n_dims:int=2, record_range=False):\n",
    "        self._n_dims,self._range = n_dims,record_range\n",
    "        self.n,self.sum,self.min,self.max = 0,None,None,None\n",
    "    \n",
    "    def update(self, data:Tensor):\n",
    "        data = data.view(*list(data.shape[:-self._n_dims]) + [-1])\n",
    "        with torch.no_grad():\n",
    "            new_n,new_var,new_sum = data.shape[-1],data.var(-1),data.sum(-1)\n",
    "            if self.n == 0:\n",
    "                self.n = new_n\n",
    "                self._shape = data.shape[:-1]\n",
    "                self.sum = new_sum\n",
    "                self._nvar = new_var.mul_(new_n)\n",
    "                if self._range:\n",
    "                    self.min = data.min(-1)[0]\n",
    "                    self.max = data.max(-1)[0]\n",
    "            else:\n",
    "                assert data.shape[:-1] == self._shape, f\"Mismatched shapes, expected {self._shape} but got {data.shape[:-1]}.\"\n",
    "                ratio = self.n / new_n\n",
    "                t = (self.sum / ratio).sub_(new_sum).pow_(2)\n",
    "                self._nvar.add_(new_n, new_var).add_(ratio / (self.n + new_n), t)\n",
    "                self.sum.add_(new_sum)\n",
    "                self.n += new_n\n",
    "                if self._range:\n",
    "                    self.min = torch.min(self.min, data.min(-1)[0])\n",
    "                    self.max = torch.max(self.max, data.max(-1)[0])\n",
    "\n",
    "    @property\n",
    "    def mean(self): return self.sum / self.n if self.n > 0 else None\n",
    "    @property\n",
    "    def var(self): return self._nvar / self.n if self.n > 0 else None\n",
    "    @property\n",
    "    def std(self): return self.var.sqrt() if self.n > 0 else None\n",
    "\n",
    "    def __repr__(self):\n",
    "        def _fmt_t(t:Tensor):\n",
    "            if t.numel() > 5: return f\"tensor of ({','.join(map(str,t.shape))})\"\n",
    "            def __fmt_t(t:Tensor):\n",
    "                return '[' + ','.join([f\"{v:.3g}\" if v.ndim==0 else __fmt_t(v) for v in t]) + ']'\n",
    "            return __fmt_t(t)\n",
    "        rng_str = f\", min={_fmt_t(self.min)}, max={_fmt_t(self.max)}\" if self._range else \"\"\n",
    "        return f\"RunningStatistics(n={self.n}, mean={_fmt_t(self.mean)}, std={_fmt_t(self.std)}{rng_str})\"\n",
    "\n",
    "def collect_stats(items:Iterable, n_dims:int=2, record_range:bool=False):\n",
    "    stats = RunningStatistics(n_dims, record_range)\n",
    "    for it in progress_bar(items):\n",
    "        if hasattr(it, 'data'):\n",
    "            stats.update(it.data)\n",
    "        else:\n",
    "            stats.update(it)\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def get_data(size, woof, bs, workers=6, stats=imagenet_stats, tfms=None):\n",
    "    if   size<=128: path = URLs.IMAGEWOOF_160 if woof else URLs.IMAGENETTE_160\n",
    "    elif size<=224: path = URLs.IMAGEWOOF_320 if woof else URLs.IMAGENETTE_320\n",
    "    else          : path = URLs.IMAGEWOOF     if woof else URLs.IMAGENETTE\n",
    "    path = untar_data(path)\n",
    "\n",
    "    tfms = ifnone(tfms, [flip_lr(p=0.5)])\n",
    "    data = (ImageList.from_folder(path).split_by_folder(valid='val')\n",
    "            .label_from_folder().transform((tfms, []), size=size)\n",
    "            .databunch(bs=bs, num_workers=workers)\n",
    "            .presize(size, scale=(0.35,1)))\n",
    "    if stats is not None: data = data.normalize(stats)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## The Transform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "Not really sure how best to simulate the sorts of differences you might see in images. Here I'll try a fixed offset, seems like that's the sort of thing you might expect from say different cameras/lighting etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def _rgb_offset(x, offset_range=0.2, offsets=None):\n",
    "    \"Randomize one of the channels of the input image\"\n",
    "    if offsets is None:\n",
    "        offsets = torch.randn(3, 1, 1) * offset_range * 2 - offset_range # -offset_range to +offset_range\n",
    "    elif offsets.shape == (3,): offsets = offsets.view(3, 1, 1)\n",
    "    x = (x + offsets).clamp(0, 1)\n",
    "    return x\n",
    "rgb_offset = TfmPixel(_rgb_offset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "im = data1.train_ds.x[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "offsets = tensor([0.2, -0.2, 0.2]).view(3,1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQEAZABkAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCACAAIADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD0qlpKWvaPoRMUUtLQFxKRiEUu3QDJpwFZmrXnlwNEhAc8AHvUTnyxuVCLk7FyzkM6M5/CrAIqhpMMkduiMTnHNa3lDFTCd1diq2jKxDijihjtB71VaVi5J4X0pymoq7FGLZZBB6UtVoJw5OGztq0RTjLmVxSTi7MSijvQaoQlFFFAxlLR2opjFoxSUo60CEdtiE1mPbLdXSyyDOw5X61embJwKp+bsOSOprCq1ombU07aGnCAoAFT7uNtUILlS+BVouCcj8avR7GM4O+o4rVHUQYraSVVZiFPC9T9Ku7qjuFaSI7PvY71M1eNgg7SRx/g3Uv7Rvb8HAEbhdm7djI7kcZrsVPWue0uxubTVbqeYKfPxyD6V0SjAqMO7w2N8S053QUUtNroOcXNITRSHpQMSiiimMKM4BNFI33eKAIM5J7Vl6sbuLTrh7OMSXCxM8YIzuIHAxWqOO9UNZ1qw0XT2v76QpBGcfKMliewHrWDSk9TZOxz/wAPYdev9OuLrWjMjGX915owxGOePTNdTreoWug6ZLfXspWBRzgZJPYCsLQviRomuX0djam4ilf7iyxgbu+Bgmul1nSLbXtKaxu13QuVLLnGcHNUko7GLm7rm2Oe8P8Ai/TteQmzmZZVPzQTYDAevHaurjYOnbFcvpngXR/Dkst1YQuJmGA0j7to9BW9bEqSpJ4PHvSvaVipWlG6LLQKcmiMHBBqU8rUZ4NUkZJtqwhpDRmkqxoKQ80vam0DCkpaKYwpshwtOqGY/LSk7K44q7K8j/KecV538VJDL4etIw3/AC33j3wCP613zjORisDxNoFtr2nrbSzvDJGd0boMgH3HpXFGeup0uF42PNfhxox1DxLBPIzLHanzcKeS3b8K+hYGJGO1eK+GtD1rw3q7SNDHcwN8heJsnHY16eNXna2YWlqZLkDISVti/icVupRta5hOi+VJGxcyr5RGR1wagCBnVgT05FYd7qN6trCHhU3LsAyqcr74NallcNIg3rtIFR7SLlyj9k4QuaSngUx+TmlBobmt0c/UZTacfWmmqLQZoxxR2pe1MBtIaWigYlRSjipqil6UpaoqO5Sb7/NQvGGJx1PAqyUpVTuRXnuGtjpUrGZLpblMwM0bkHBU4/OsddN11Ll3TUWEakqSVBJ7jiu32qI8HjioIolCydME5NVKjZoUa7OdhNy+1nVpJE6t71t2wLRhlbDY6VNHCoL4HHWnxxIUVgO9TGk07hOqmieBiU5qWo0XYeKeTxXbFO2pxy3EPSmGnk0wk4q0NC9qTtQDkUGmMSlpKWgAqKX7tSU1xlaBrcrdqcuRzQBzinDpXJJamrFLFlJFG3ERA6scmlHyACgNgc9abV2T6Dvuxkd8UkRwuPTinDPfv0pREc5B9qpRZN11FU808dKesYANJjANaozbGmo26U8+tNPSqKQiHIpxqNDhql60xvcSkpaO9AgpO1LSUAVujkU4dRTSMSGngAcmsJrU1Y8ckHr3xTgFXlutKqnbk96TZubFUokXF3KWBzU6EEZz1qEooP0puSrAj8qqxLV9idmxk1Hv3c00kmnKoApiskKRxTDxUhqN+lA0Rg4k9qlquasLyKa2KkFFFFMkKSlpDQBXm4bNCnoTTplyVqIPjIHWsamjNlqiwZc9+BTVmweD071XMnzBB1pUKrnPXrUqTewcisWfMLE+lKvyHPbFVhKQeB8oGc1LHL5kecY+tXGVyXGxYQ8YFPHA5qFDxwM0ryYAAqzNrUkNRMQRQXyKiLUmxxiNJ5NSxtxVdsmo/OaPOalTsa8t0f/Z\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAYAAADDPmHLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIABJREFUeJzNvdmTJEl+3/dx94jIzLq6unu6Z3Zndhe7ABbAYqEVLhllFCCDgaIgyiQ8Sf8aTY+iySTTK/VAUSJBowwL4TBAAiQQEDGDPWb2mKuPOvKKcHc9/H6/CM+oyKrqnp7qcbOszIrTw3/X93e4h8v/Q87sa270bS0Xn/GxU8ffdP3r9qXinvvO83pc0n1Bt42Pu22/xvd7kXP2jdm+e6c922+6z/iabrRv3zmjVr3E7eVm5cNOdeizNDdxj7IlpOcB6ID1FlICHyBU4J0cY0zgJ/pubd+Avcpn+Sz7b9tuYtg993k5Bpi6oL9FJ27bbrqOSY0H2o50+RSPg8UB1ICvpX+mEcbaoGSwV9Xnsu3TAl/A9tkYYCz5N6mhF7n2dYPn9ROB1XMul885qGvCvAbfyFNlIGfIbtp83ET8z0q8McN9HppmSpPtu96e7X568y2bm/hc14mXbWMCVkjPty2XF8/YbJeQI7goZsE+YyqP+/iy/Zx67hcc+FfSXoHAvZgGeBFbf13nXmRQysHNCOHN9l8+J60umKVIyBmSErxC1H8AugStB+eumoOpfuwDsyUvlX1KiCYq/w/FOTBtHqfGcnzMHZiQ6lZIfN9A3HRsec6LIFYb4LENN2Dngc0F3fmnhHbLQaiGfiUFCB41BQnaDnINzg99ccW1yr6M8UJmIDLF8bYtFs9n3ohpqannuw6AjplsX7uOUcrnG19zor08BniRtk+qbnrIseovmWJ7yeriObldQ3VQEDMPg5AzuAwhQPbT6n/suo33GaFL5gPYdMJYLkBVCdYYu65j5rJ7fB7A8yXbizPAZ1FLt8UIU3EHUNQPdEtoL+m6S9rlBQc+Uh0eiLTHDtoNeK8gMEPld68z/rbf5X2NiOO+JKA7h05dz7oR4OlrYYachdkyYgpyce1Suve5uHfc7kYDwMs97JR9TFtYPoH2ksCGs8tPuWTDveMTiC1svUqqVybwAwPYdcpPGt3DNIzhDQssRWCdIZ9DfgK+giYI8ExboALXyEk5QGwGsFqaiTETvOb2YgzwWYk43n5T1Kq0/0ac9gKWT9muntBtz+naS9qtg7iFGKFrBfTVlQC/EAZQVl5/ivhlCwwmJ2ZYX8J6CfkS5kto5oIp2ij7qWB2BDTyO1WD2dlH+C8AE3x+XsC+Nrax+47xxW+HSFG7hvVT4uY5zz/9KWeffkRdORYHlRCs8hCcRAJ9EPVc1xIcKp/BpDEh3sRUaLvT79jC6gJWl7BZQt1B5UTqM+J5ZCff7UYYDyB1kJqBkb5ghLf22eIAn6XdZgCMUQxtd0tYX5DTis3yOavVOfN5zeHRkQAx7yUUXNlHie9HnzD6LkPGRqwuw2olxF9fQFwCLdRZGCB20HXCbFUljBdbeTDnVFuxq2W+ADZ/3KoriPW6dhuX8TZtXwRxjMYNfUdgdQbrJ5A3dO2Ko+MDDg/eYRbmUM2E2Cgxav2/fKapsHAu7mEfu3/XwXYN20shbOVhNgPfiQfgzKvwEBNcrsAt4FCxwAZYA3OuAsKpBNeramM6TgWpCm13dyDwRVrpjmVg08LyjM3FU6qwot1sWG/WBFdx794J3HsE9ZHYexPr4Afbb0CuNC2lHz/FACBEriuosoA9Wg0oVUN8YbsShth0cG8Ol0uY1RAWuwS/bRzkjs1D1d/0C90SxJZ2u+bZxYcsl5+SU+Lk3jH+4D7MTiEcQGhUMqtBxZfuVunHG+FLTFB+qgrcTLWEB7bQtiLtUTklJbhcki+W5AjeNeAXMANShBRUm7BrXqztI/Yd0uOLqQFKExEQ+1o56spxeXHOxeWKe6en3Hv4GBan4A+hORQN4IJIaQkip3x8k347pjwevaefyYbkoE30CabgxESsVrRnTzl/8ozUZY63HbPZIZw8kAvmJBHIUgvsC0WP3cTr2itkkC8mA8CgkiOQWsgJ5+Dw6IjsMyenD6lPHkE4hHAEzQG4JBrAYu9TAMwkvtQA5TE7BFAODDXkBqpG7XmEbQsXz7m8eM5mfcZ20zGbz5idfQwnJ/DoHbiMggVg0AKwa6dLBr1tcOimCOoLtC8mA/Q+P9BuBYx1LTlnjo9PWBwdsDg6lQica8CrTx4cfdLHgi/GDGNcYdI4rsgppdVBHwUKQYBljJA2QEdOG6qQqYLjfH3G5UXD8cOHkNcwcxCDFKtUzW4uYSqW/yIa4BW21+cGXtcyIjHbBNuNMEFOeO+pqpqTNx5Tn9wHV4vqrzQZ5N0A/EoJ3we6plzCHgPoNmcb1MXMScLAHlxwVMGx3V7w/vvf44MPvgf3j+BoLt5DFaFxQ8yhxBxTuY7X0KorCRCYjpVbe9GOTl13ygaW/ycgbiCtIG8grWlzJIYZVX0EzTHMjwSkhbkQq3QbS5tu9zXitnpcVdwrFL+NGULxP40AuhwgdCLh0RO3FR98/wk/+Pc/5m//5if81m/9IvhHcF6JRjqowOcBb5Rp4qks3W2Z4jotcRsNcqMbOFZFryICOHWPkhHG90utRtu2iAr2hLAgVFr1Q60JGDcQyvLyYXQt+44Mkb/yecb2tww/9+YiiCp3R3B5xtO//3v+/Lv/Jz967/scNQu++s7XqfwM1klyBKHRPjkJKvVcOBqDfebguvYyZmI8vl+IOMCUS2Q2uOvU7erIKVFVFa5WVW8nZsQVszw/itCnrm2EN0m0ASmJXZ6Xim9zH+tKzvcV9995m1/8pW/yzuM36M5X/Okf/zkHBwd8a3MO9QNwebhGHAcCRs982+2fQ3t1DHBdXH/f/rFrZjay0zBrSpATOSdcL4mKolLHYLh1u9eLlmFfu4dJ/1hDjKOPJWNYBK+sQqpnsA0QHF9+6xF/+O/+kr/44/+XT9KSx5v70F0AW9isIB8pKFWOG5vb8vcYBN4RJvj8NcBt1Vkv+UgQxSGJnAy+TeTcyfYAfd4/B43+qRlw1e79bIA7dos6TBPAboi27E8ZFTTz0iJeQNPAyTGeR/z2f/v7/Pbv/AP+7z/6I87OVpCXghO8ZiZdI9nCyl3VSmOvBO5U+uEuNcBUG2vGhLpZ6nbZhZPHJYTwVAMDEMQTcMow2QlYGxNvHIUrXb2Oq4QxBhkTpa//04TPrILLJcvnH7LcnFHPPaAMUHux/QkJHlVuCEdfN17c8phX1O7eDSwlrfz0gR+x+6QoAxc7iJEcO3oKpE4/RTA/JzEZKe5KvF3bkL1DUrdl/D9ydbDHIxPQOj8H61byE8+fw+YSXzm63LI4noFroVtL/2JU8+F33cyxFzD2iu7QJZzWAFPItNz3Mh0cq9lx8MOIn5TAWanTtaQYyWRCF0XyYhL1n8OQBUxaFu5q2d9qQYhd2wjaKaOYbR6DvqlntlYBBwvgCJ4vYbXh45/8lIvnP+XRlx5QhYZ2vaQ+fw7pAKog9SGBq1rGxqEMVN1mDF9xuz4OMObIz+J+jFH3WB13qF1PA9hLknhxLuOcI6WOtNpCdoSqxuVG7GxWCucWcoTkgRqquRAcRCXDoCmsZtBrR8aJoRIomqLp3cQZVAsIM5rZIT7MSfmSmBzrzZZ6tYRqLUkhCwJRXHs8nmMgeNMY7hPIl8APd+8Gjh+yV8VRiONVLNzgu2XNA0iUN9F2kRShSWtRxfUcmho2a2gDHBxIqVbltEzcQdDMXs5KfB0tq+bNCFjLsDO3EHYDRQ5ojuAogu+4t6i59+gheXXG5eWKo5OHMFtAM5M+eQYXdF/gq/z/OrfwcwCItwsEldtftI3V/lSLDGq/l5Bc3FPURM6JUAW6riO2G9YxUrma0CVc61ht1ywWB5KLb9ci9T6Izd9qljDrDXLW+znNITAwiBt11FR1ry095BNh1qaGk3u47SVHy5UEgPrMJAIYff8gu+P4IkCwZIJXyAivPxmUUFCmGsACOaaeq4DHk7Po6NR1ajYzlXNUizm0kc1moxigkzBy10GspE4gblXSbfZwKMrHN5Ab2e7SEFDyKvYldilDuS6AO5RikaRRwnv3AC/p45gliTXX8rS0JxB0UxvHBu5EA9xV69GwATOTevXfnPr5OQjjpwyVx+WW2GrBnXeq0iPOJ7abS6pui58tFFNoQCm16hJWg7rfbhW1b8W394rWIlC7oY+lS2AxARAskNE+6oxknwV3dJ2YpLoWMPgy4HlK9b9iD+H1MYD52uYn7yTn9YlTlHBwThAyBE9er8kpM18s6LYd7/4/f8V7f/c9msOKb3775zk9PWWz2XDkPW59IQRIJtk6gweUSBHYwnoF6xpmjeT+k0btfKCvLxjn8HuTMJPcRJgJk7kkmshvh0phVwGL3ed/UWD9OQWIXr8G2P2h/2YhTrdhu9ngaKkbD7XDLeasnjyl3Vzw/e+9z//yz/8Ff3X2Y77EKf/NYeDXf/3XIUPbbvAxUR2dIjbfCUHKuEMXIW5ZLZdkYDGf4xotKwsemoVIt1UWl3WD9n9Arh+0JsFFKRipvDBv14HbiNfwMu1zjgwKA5R2DnZj6Ldt1x1fXr8siHBAWkuBhW9FfToHRHCinpt5IKZImyN1qtl2mVVecLHZ8D/98/+Dvz77KV/iq/yIDzj/yNOkRyTnqe89FEaq7os30CYpzsBJSJcZVI/g8C0WBx2whdDCJz8Cn2B7AacPYHEMXU2fc67l1B1GCqjJmCm4jOKS1l6KRqNOHrF6BdMgJUPZ7zJQVP5mtH1q3PeZGXNFJxjpbnIB+zoWNaTbR/l0kLwT164KQpcuQYTttuPpszOca7hcrnl2tuU5Gw45Zw2cXbSE4xPCg4eCC9oOjpyUZR172M5hBVxs5f6Xaz569yOca3n0lcfw5j0Ib8EnPyGv17jVUlB97SX1bNHEyG4RyU4zmG71iRaQygIEjWGmytH2EfZFx/sF2t2agDG3piJ8u1Oh6dSNqgWkVRCqmrhaMZ8fcHz8gLb1PLh/iHua+JRP+SZf5Ve+8y148y14iNTkb4M84ZxB4o6AdeD8+9/nX/6v/5o//NM/5vTomN/49W/zn//j/5T68QlUDa6yKGFUcNqJp1AWipSh3Vg+l4q5S0MdYUr0k0b3EakEfVxz3HVt6pwy5zKCWbcLBZcdfJk2jihap2z2bu/fqj8edR9ZVHddQxNoIlxeiEt37+SUX/0Pf5Xzf7uGFPgvf+8f863f/l0hsEeezBaKiEhIdg48gyf//v/jn/33/4y/+sm7QMWTi4/Jf7Ihxg3/yT/8NR48OMLNKrHhm7V6IwlyNTxDmXCyZkzmkWikU62RNJCwL/MyFRHcxyi3ocH4vGvoeXfrA0x2Sm2+9yJlQQfJKQVTJ9+hFkmaOQ4OEz40HL31iN/7vd/jl37xVzhYHPGN73wH3mKQRgvn2m8jzuWWjz/8CX/1k/+Lp0QOOKFhxo/WH/EXf/g/88MfvMfv//4/4u233yLMGu17hvkhpFpm/cJgs3cCRMWzekQTeCfMYwww1oLXaQRG+14UEObR9wTzvJgG2HORa1sp/WMO7+O7ygReo3JVI0EcgJkmVZyHakYdtrSblrqJHJ8+4Nu/8kDKsN9E/Xd21XLSbcpLPGx49PghDdCx4UM+YUEDbFiz5t333+eTTz7lq199G+Yz6UO7ES20XYrnMFvs1g6ObfeO3+7VqyiOK3MDtyV+ef3bRhDHx04AyOuXiNl38Zdt485h/raGS21mr4VqcTLYlU6y8A5fz7UoSN26k3ui2lu9dqcfk05bqsXq8g/g+OiAn3vwczx8Hvn7+IQLnlAR+AZf5h/86m/w5S+/o0QLooW6DTAXb2XTCnPOdMpxGD1TiQfsmcvy85L4U2bkszRjvtKbGLfMjjBWn1eEaadT4w5Y9C8XKyekKAkch7pNQQIzzqsNRZF4IwwRM8wXQvyy3GvNQOyaYTCspCtCvVjwT/7Jf8HR6Vt8cpb54fffZbM+42e/8Ta/8ZvfhqMZnH0MZ2fQBLrNimpWgz+QoE/W/lV+QPU9dmG3NL1U82PtMHaJb9IIL9LK+06VJOp9qpcKUX7WltH4vxZyOAOD1tM0JFPQ7YakFwu40KTLQbNDWDbAJUMa1pB6+YwN8MYbfOvbvwxvfIWvHlf82vY3YdnC3MNRgNU5XD6lXa+pF41gt81aGHKxUJOVhzR2ZpqYsEts2zfe/6oIP4Wz9mkBbXfrBubRP0mdYZc1I6cZu1op1xeGOJXyCNsO6gUcH8IywtNOpm1vkP/rIu5eAdviKR2iFR4vILw9uHVvIABvhUz5ihHmc+q0hfkcbyVoSdcDyHHQSibuhj/gqo9vzDEF6MaBnvL7DgTz5VYIedFWAsH+GrmwRSUYVIawDF4EYQYt4HAemhnv/clf8L//qz/gvXd/wHw+p5nNuH96yi/8wi/wy7/yy3z5578h0ilob9AKGThAdpRao9JjnU7t9scK/tQTePZMAkvdVvpj6wO4sLvQhD1fad9NDV/nnt0gqbfaXu6/pWa/Gw0wRqS98OiomYonabzeUJyDNuvEiiDLwVUVf/unf8Y//af/Hf8uvceGlsPVAYkIP8388d8+5Hc/+M/4r2f/FfWs5uTxI3igUlpm8ub6PdNtM7kFl8Ay6BpATrDGbC4pXbxoh7BFJqakwbUbzy+wNiaCMcI4NjKF2u+gvVhRaH7Jz75mhRdZo4GxlRKvtIG8ES3QroibJbnVRaA2W/7gD/4Nf5n+mkgkkTjnkkRkTs1HfMS//fM/4IP33+fka2/B/TCwualos9kg+9TbY4mYDA87QaqqEU3QNPSFqn2dWKSfyFoGM8swsf1fFoaWGsOIX9YlvuxYT433NZrgbqqCJ8FJHkrA+5BwWdm7hs2S1G20flN86Seffspf/81fM6NhxoyGipqKIw54wD1OOeYnfMx3v/tdeHIJz/Kg6i2GXxLACGPo/QBY6Nw+K+3yiiuCFpJsllLsYUWsMe+uNFJe34+fm2lkPgaFr2q8bzADLxYHeNk2eQ9D/VHdQYvZqnu4bUltJKVAVdeqgmG9XvIRH7KlZc2GRhfju+CCS54TCDQ0fPLhRzz79Dmnjw/ldibZJVArU7tB97XAUgtNZrOhgshXsFYM4LLUJyQnOMUKS0vwZr7/OPiVTLP4qwEkO+5KwGzPcS/SppiNfQzwMhy4j5HGIVKTfrKGatWXcvrJXY+yfYzENpJTwIUOVi3LiwsqHd2AwxOIZNUGsrZbzYJvffM7nB4/lriASeKWITI4ljgn3aBFGNPrSTFLZvL0EXz6qWCSowOYn0qImnq4nsUfSikeM8O43tA0xJjQU8DRcaNEX4k2jreX1+d1VQWbz58Lte8iOPvfJn2Ac46cM67rWK1bYtdxn1M+4RngSEQqamoCLRsOWfCbb/0av/M7vwUPbM0+9g+kKR5b1TOjE1O3wgR1LdJ/qJpktdSchXolKckaQiUQ7J+TXU1j976tgL2scN7mGO3Hi2EAd83nus6MbZ8R3nxrm0VjmMAqc50bBCYluu2WzWbD6eEpDZKUaYlkYCuiy89UX+M/+o9/k4ff+MrQ55YhRGzWpmzlvEEj0HpNn8dvt/K7qgcw6J08Q9cOoeexb2/EL7WBaYExo0wBufHYjrXW1Cddc52x5+HuGgT2HYtXNUAuP3qCc8TYkVIHOZNSYr1ecXh4wD3uEfDUVDgcCxq+ymN+5uvvcPrgmHz+BM4vYBV3iV6qWpP4lsE82EDZhJPYwWYjBaQ2x6BphuVobJrZFAGm2lhTXNf2aYDrJPxGz2v3+9WtFr7vpjbgJQgy4Je7guD6MbWaIqnr2Gw2uujngpw6Ls6fk7rIEQecccmcGkfFEXO++uhtvvWtX+RLX3qL1XrJweEhHIZd5F9O99ausGVggITUAZhbmm32cRo8AQN9XkHgWM2bNrlp/MZu3/i4fRpg6vjx9jGmGF9b998dBug7bFJugK/VTxowQOzIXSR2kdi1BNcAGUdivVyyubykxnOPe2QCFXNgxdNnz/nkk0/x3tM0jaSRz5dwcjCdjrV+ZVTSkejf6gJSK4s/VpVcBwrAphQOxlla7VN6FVPtsyD4krA3tTGjTEZipd19LqDPAsbe7ucccVlLr1KUyaD65g/nHCE4CE4KdWuASIPnTe6zJLLgkOesWbZrLi9XrNcb4pOn3MPTPLgv9zb1XmYOjX69NrCVP5eSAk5ZcgRNo9pAuSUhmUBf73oRU1k38wDGIPS2QG3q2Oukf0z8MTOOrjfNAKVrMr7hTap+qjOlGbDomUm/jzh7s5cCQO9lpU8fI0dHhzjnSetLQsjMZxVdu+QJT8hURBoiWx7xkMiS9XJL7jKzwxmV1fWvkxDMKrOtUighqr9BKobbS/HvfRImAI0bmP+vQNC8gHYrE1BnXtxNG01babz0LIxByhSxK/YbkLT+2VDtG/MpoDh2t8tte8zB3ZqADH2ghyyJH9Dp3qPqiJxIMZJy1nc/RGazii+99Qbf+Po7fPjeU37MMzKJGTM8LY/CKeTMJx9/orfL3GsqnSya6aeDnzMM1NPn5OdPcIczOKxlZfCnH7JeXdI0M/w8iDkAXSZOYwP4gWmTFyYqC0Gm2pgQY6JMmYgd4Xn17XYM8Fk6YIMSYJgBpKLgETNANzCArcsXEylmMhHnHc7XuBD42V/4Bvfvn/L1P/saf/TdP+Mv1z9gw5Yf8j7P46fMfuT54AdvcHzUEA8DRCV+PZM08gyR1k9aPnr3b/jxxz/kGz/zNie1ruezOuODH77HannOyekpD3mTKidJDjWHqtIbDVzVYjbWXtLQU+N1U0DG2rhU/DoQ9wrb7WsCP2tnTPopMECfTEmDi2WOvwuEEMg4fPDgdXEI73nwzmN+995v8/M/97P8y3/xb3j3h9/njOd44CfL7/Gnf7Lk8Njj66/TzCL1esbs5D5UCc46nr77Pf7V//av+bvvv8vp4xMWs3/IyYNv6iygjMtbPvzwR3TdmqpyHKVIMz/CHUapVQi6WlilceW2VVwQbnasx4CsVNVmDkYV8p+X9AO4/D9e8/LoF2n7MECpFnML+VJe+5KWSMZvI4Ue5mbZcux4hlU/LDqY6DYdzlWQG8LskKcfPuX733uf9/7uXWLc4ILj6fkzTk6PePDoIV/56ld4/NabeGpydgQXIHl++uMPqRYzTt64z3Z9zoOTOa5bslk95YMfvMf582e88eZjHrzxJoujE8LskNnBMRzdh8WJ9LM5EAxgwLJqBpEyAHad5zHls9uY2flT+GvKxufR7+swQLH/eg0wtlcvyyq5+JGLDwhIC7nomPrW3ipps9rZlm67Zb3ZUFdz1ssL0vMz7t17yHf+g1/i6197i/X6OReXl1xsLqiagK8dXXfG2fNMt42k6Hhw/yGP3/k6p9/+Jqw3sLwkbQ9lYfDzjpAbKg9f+vIjHj56yOLgAFcFCQcgcwnpomiLtpOS70pLxfNYCornn9pVjvMk0CsOmCL+K2i3xwCvpJWRvsxQCqai0kfV3OAyOgkK5RSJqSPljpRaZrOKs7MlH320IUXH0WHD6ekB9x8c4ps3Wa6XLI7muMrTtp3EdTpYzBawcLA9FzBXBbyroE6sVxd02zUn9455cP+Y+emp1B4mPwSAbA2DuhYG2Gy0NtEL1jDJLQs/9kn62E27wigmGG7n31fZ9q8QUrbbBCHGCHbq2OQg65Qvm2Hpndp3pVDSahtXUxaLxpzJDmazGS476hA4PKw4e3rOarXB0bAOCe8dbzx+xL1DnVvoPTUtmcxqs6bxDtZncPkcFodaZ7iCLuHyhsvLM0Lw1POF+PmuUuLXWhegLqLPwrjZQbsEp4tMJB0MU99jEFiOTUn8ybCvH/Zd55Lvi/Xvo8ONJuBVtlIaMgUDGCBUo5c7qQRiqw9Ta8JFTYLzOOdl0RAnBaPeRRZzz3w2p6ortu2GDz/8kMvLM+oQiO0WcqZbb4hdx/JiyeM3HvHm+TkpJsKsYeMAl3Eu0XZb6qrCuUBMjtDmfl0C8dW1r9ul9LeqhfA56crghwOTl4BuX1xlrPpvCrPvMyV27l5TMvH/nTHAvuYQCYpZl3OxcLCG5pwXDeAD5AofNGcbI66qoOsIPnBweKjRwkATZ1xcLnFkzi4uePLxxywvL1gtl8yqmsvzC977++9xsFjgkmN2dMDDLz3m3ukp9x+e0sWOg4MDtu2Gtm3xfiMFE14Xp7QVyHyUMvFtK4TP2ueqk3kDsIvkp7QAXE9QO3ccB5jCZ5+h3R0DGMFNqlHJToWU2zo+Xt3BXqV5vA84SxNXleAFF1ViPW3X4asZb3/la1RVxScf/YScMwdHh/z0xz/iYHGEc4GnT59xdrnk6OCYNiUulytO7t9jcbBguYyyHF2GbrMh41n4mjDzw+oheAlc9QkfYwytIZjpOgLR7S5jP4XMp0LHU+N2Gw3wku1uGKCviFEi48W+WZm31/RcL2Wga8MWoNHh7FgXwCV8CDgt2apchatryIlMxFcNDx49ZjGrWRwesDq/JFQ1h8envPn4Td549Jj1ZsnZUmz+87PnXC4vmc0amqZms+3IbKibOcE7yQc0c7l3F2WJmah9c0i/uq2WkRuFwlUTeE1YdrJNMc4rbJ8/A5QcbBMlbQXPqINnICtmXdhRnzqlXbfRqceQMviK6kDfGRAqXJB5+JtnT3AOju/dp6oczjs2bcfl/JI3357RhBlN3VA3M/ysJlWC1NbrDTnBdttS1TXOexKOto3MWlvCzokmOKhhs4XLS2wpW3mRVBZTpuATGJB+Kbm3IeKU5zDl03/G9hrXCPIFMNSPmQLc7kM6NyBwB7mLuEolMnkBY7MDSJ7ZcYZ5I69qoYPNiubZM2aLA06+8jMSEn7yHNqI31zSxhWQiDFyeHzEYjGnS5k5ToOTnq6LVOuN9jnLtata5iautVCk09CwvbLeZV0tXMPEJfFLE3BdK9PLrvh953GAV9HKSRlGYHvbR18mZoQuAh8Gob2ZDkgpSkTPV/QvjPAlu4aIAAAX0klEQVT6Vi+crhC6lt/NjHsPH3L+/FyIU8/g6BjWLY1LHPljVus17XpNM5vTPHyLJiaR6q5j23ai9mOSqqAuwSbC4ZGo++zludpWp5O7QVNZgWsO05I7Ru03EfYm9+4l2ssxwNjnH+/b27mBiAKiasm0ZRjetK0ltkl1psuDR6Bv4QizhUhgygj6nknlLg3Ma1m8MTp56VTOnJ9fEppGfPqq0Xf5yCJOzQZ8teTg8IT6+J4cczCTAM/c06xX0s9GX09jlcKXK2HWSuMDOGFGj/QrIOYi7z56nxIej+NY5atjcSu3rtx+E2OMtO6LMcAUB49/T6HUnilMtetoKKDLBBxJ7abaWiu7cmn4zgzLugYlpqtlOTffAAsJ0PhWgzuAd1TNjMpeJh103cAAuAXUUeIKXdSFqWr5NnVupUSzRryPLg1x/1Awhelph5xnaw2Ox+FaAZloY7dvisg3uZLXtNeAAQr1r/be9a9UKaoynY6UDShWeq2lWKHRj1bqugpSNRRregWIwOLwRJ9WJ3mEalh1xGXRPutWFnus5prUmYufH+aK7meCLdbq0zVqcmISiQ5abGg1gkD/npt9wO06Rhjjo32RxTJO8BLg8PUwQB6IP3gGOjUcV6B+O8UYQpsRvk+SOGEAe3mUr0Qr0MqiE/OFBpsQpJ69HBMAF8Vk+IJpfCP72+WgusNMmMCmrPtGNYWmtL1mMLMfMMx44ekpyR0NzRXCT33GLY/OvQlrFO3ul4m7Anoc/SJRRtAUlZDm+sGwdAwiwc4NLmN2QhCryUhecEFqJG4/nwtat6STC/IGMBDgGfS1M7lS1V8JwfNWCG4rk7hKtY8WgVqeIHUDDsgKYpMy99QYwG4l8VhD3GTLS03g2b3O2GSU15645utbJaxHyyZiCvJM8l1SBmAAgqCqG/pXsVWahClHsB8gW1wqSMlWNdNgTiMgsd3oKmKVagRlAFcJQ2UvawIdzNXjcPLt1EuI6NxB6EPYvcoYieE+FT0mGLCTBi73TWGIxH6weIt2U/3K59T8oO77h/QDQ3gv9jpoQMWmYYG6XlnXEwqwOBB7nJAJHokioKQM0moFsmkbnPxvbwR3XrSArxXVV6qVvJzrtR8ZNVUajErKmDaPsAx0meZ6UdtfNjf6fA6tmuzMy9xsyp7Zd6n2ejcwMSygqAVxniHnjhKYpLkBvUGOIv3BAixJXEmXIQWJxLkNxBXEpahxj9T6uw6O53J8txY/3VlwJ2hFcKdVw1HO7XRW0EyTPAEh7mxG//4ieyavWsDsvmFak/4pJG8iWIK78vyUB49oR1zzkHo2kLgvSLSPxu61RgLdIC3W8wxXejueTWvED+aWdWo6EILGLYRiIqDLguJzN5iWZCld0wgoCNT7WxWSy0NiKiNM4ZIuYKWUjd1VdG5YwwDtddqgGI69oV/joB1soDcd440XFN67WyJmp5kHwDC4UNg+VDJ1VG0lMfT4ekb/trAYh+OT3syCSCGIW3i2lOMXc9E226VE7lBTY2YmBGEux+DHz2cK9DTc68OwbmE2j8UNDOEKs2IPb26sPbqNyU2um42PCYd5FrbCWkoDE990vS+EF9A37WUuWdq4WAcRBlNhUukQO23vE/CK6Pvp5oUvZIkkCzjN5jA/kGOsrt85+jeIhWrwLiyql7MCTOR+Zi66buif1/S1rSJSprHJwznFpknzaPv2NmU0I/7O9luCinK4tR+fPwOUOe8dyYAYI95lXO/uBfq3cqbxaguqVkPQaeRJ1uivTRMUNzJ1nRT4VY26hXrdGAvAlpWIqtZzwTx2za4TfDCbi4bos5SF9upr+g0YIudm7cMUoi+/7fcVWpZaUX97ZYRe67yg3i+Y4PWYAO+wgElGn8tcqV7tFaNhhSRmr22wU9bAjhLDpNqijFHDytVMmMveIZjdgOydGyQ/5yExZVJl7y/MwEKP9yo+CT2n0/MtZ1Hkf61ffmQCSnA4NUYwqHlLLvWRUT2hX11NayjsRVe3pcmdgcBJ2+Rw3uNdGgZ/R1UWI+UYAGPKavPU7rXbYZCrShglBOj8AMbq2YA1dJ1BcqvMUdh9WxWsT0Q5uWbbDcQA+nyFrRVottn4tu+6mYPRw09JfzlWJSbqK6iNubUQZac/JYi+gQ6jY24/L+CVNhlc3/vPhmhhB07n4mME6cGUEnerrpy9bq4uInPVTK7ny6SNQyp19MLm7/eEZ1Cp9m1awF5G6RL9PEObC2CxATNhrqSy4wpzjJ9vPNa5ON5UfakuSpxRLmlXnjdmiB1TLD9vNzXsNoywj/vyeL+ps4Ig9vbO/maOPipoIKLHCJq5MzWvr5ITJI98d/qEodFZvRqhi1lwQdJrmS/dmxa9bjDg5tXt1HLvttM8RNDz0bIwBhcyF89YSlzhGfbjYY83Hi87xxJbY5VfhRHRdYysNqLiKrH30PD1eAG9nS5EYIcBCinSqeLiogXlBz9IXKkecxoifrqsXL8kGxTEUDxhq5XYKPUqOw9uYWZIHGULXlncv2Dc3iRoWNm8gh6flM9ffNvvsm7QGGpsFs0E9OYM+mXnbKGt0iW8ySRwl7kAoK8KUoKJ2Sy4vFe9+sf7QlFYutUmiapE94ziZF/XaSh5Nqj9nUqkMuRc+vIj/emK8G9QzNHGYVqYJa/6ELB5EuZOMi3hJdGLUAFQeBKjZm9R7afTFc/t8uAaurTLSHbtazT43RSFlr+NuM7h+poA21mADyNeuVR8iYh7dF0PjNKrSb1RRtR91JiBLfrYT0iBvtawUy6xXEBZgwAM8YEs3oSZCpP2/rW0fnjO0gUeYwB7XF8ct4MPCtt+ZfwYGNC0Qi647ZbSD3c9LwBUC7jB985qtyaDQkZ8JSwMEcA+2mculv62Ag1L1rS6tEtl+X8nTNHp0u/2TkGznyaWUSW973SWe2cGBsNLEiloLUBwu2sRjRmhJHBimkGguMfYPDJIPlnT334wAYZpyuva9V4rBpiSguxVGrVo0lwroPfHyyBHKYnmfzuvRNFRNnOREUAXRwn3pNstkJPUPnsYfHUFiga2zPXs3wEYGF5KpdVI9rp5WxLGpFofoe+XdaVU0zs5hGK72yfCqbiYntSXoPvhmrcMCbzGqWHWcXWxShY11OtgpyAUBpcsZbXFur+M3BkhDSxudYmXHCU+0KvKJNrIeXXv1LRENSM9DtBcRKOVQp0ykNe4Q6EQdght32PpL9soqtt/5ySuallVVJrMvlBGrzGOBt5g+61NrxW878Tb2BU3+p4CNiZxKUAMDOsF6kPbwDvTEDoI3ml4Nw+AJwRJzfoZLC9UtWdJ4c7nEhOoVU22G/kO6kbh6FcBqxoZ3I2GoDPQatLJIYtFKQ7tpayU7MSgGUo4Y887lnTbZtcxic0RnK1IVhfSTdFn1VIWEreAUNbxcmHHmu70a8SQrzEdzMDFwJBRs9Ezc0DxvxvUtxVieD2+k9VEqWVNwb74kzyYAsMVfYCnkvi+uXxtoYlqLQXXMe0Bm602Xg5qyeRjySvVeh7tL5lkSmLHLuDeMXQDhrJzbq0B7qIV9NvdJhuycrezQErp59o2G4s+keOKBA4aItYp5ZuNHGPz9FYrOdcHsZOW4atq0R7GUOOYQGBYvMmWlXMMtp7R9xTYK583jY6ZGpfSLTbJLonvprhrdBNbyLI87LWCwKnmoPftdzb2O7nCxqW/bjl8s4tZXb2UdJbOfMjjw6BKcfo6OjcwkqnXrPuDeQJ6itn2iJSdlQxg6lyVjfSTq/44o/3jR95p5gGNbGcJkkGft8BA/cXzVSC4x3zfrRcw/t37+46cne4qHtLcoTLtakGRPkOmRDc1b2ZgcSBaYbna1RZVJTYfZL+Fcs38BDUHZmudG+y/FRrZwJaSZaagRP9Trt6YEJMMoM/mJ6g2NgueARgaeMy3cANUtl6PCbCBUELm7PS5/EBspy6acXSGfu7gjuQyeAVeB25WScZvqZU/B/KyKVHrYQCBUV3F3qQU3oMxjLXS/pfPsLMKGoWp4ioDpNG5Y5NhGsg8m5JbyqDQeL6BKy6QkkQ4y3Hvrz1q+XWbgP6HI2fTscXO/kHLh/W72/q3inohsuUC2k48gUorffs6PkPRhp4tlGtRv0KTjImsc093ij3Lj5leI7wthehQl654DttfUqDXRIXXY32x70mzYLjBbrxH30+0l2eAK4j1huPH9tCu4Sp8EHcw54RLruB+A2RFeLWvr/MDIb1G5CxFi5OJICgQ9JVIupmObRyAUkb8+uDkmLYV7Vk5md5tqM8xrAFcwpbS3pfQJSL3SVmu1T+w3/1Zfts1bPEMQ/V9DsCYfowPlGGCadCspsDv9nVMDz6rBiiZ4DZMZ8fsYAKJsDm3pY9kZYdMuSpGpVSboZBkMx3ODbV6leb7QxB7b4yTNFpoLiNJXT+GZWeiimYIcpxlHEM9OAlluLd8JmOEXgOo6kiFVAPDTOjio48xMIFJdDlWpQYZmQdrXjuVOsjNNF36e3yeJqC0P1Pgp2/l05s6s82FOsvs/u6Rr9pum5RpAaS+ll5Lx40IaEbPQsGdapA+fcsgYZ1WAnkFj/2cP3aZoHzmvsuJvri1NxmqgcYu2r6xM/++9/PtHAV/xpw7lVRpOGccABrf85WCwCki394UXdPMJdKRtVlCUUcpFGjXAF6vHTIkfeGkqdJSfWaEQWqNuJnEju2sAcat5giaGX1gqLBUux5AIb2O4qDMkEq+RRsTf7djI1xgDIGavA5yVfThanu1GuC6Z7r2eZ1Klx/GyfIEVlVrRZFVpWsLRyFEPdOQaB5q/4D+5VNlGrmv9fNDAsf5wS2EYZ/X7dmKLKLOJcyiYXI1CLIxQyqIkQuCe6deWXFvG5NSOscmtZwS11+TgSmupIwNP8WBCUuPcGy2eNUaYHTxHem4rmXPzihYYKe3m24Ad4Z4gxuCO0k1QqUunSH4oHn9thtCxlm1SGVVwhoyTkpoKy+zMLWp7f6tZgy1A31OATExY4L4ou+2xI0PV1WyPVq+5v+9Y5cHlb+DM7KMYYlRetMyXPvzxQAv1Arb2D+U7XIDQVKiX1XMeV1eNjEUZjj6+QBeCVTpNRJAGrKIwUmNX+2gDYNHYeagn6OQhvoAB/16RqUk+nHfi/02qaXEN/YZ4bwdQXLFhiulYaPt/T11f4mZpjSMts/GAGOOnWq3MXVaHdTXWZAZ3t45cRFz+bo45AaqCllmnmHgrGoIRMpbteW1VfAAc/22tHBfqGrPqDbVOS31UrMSVb2ZabKC0syuBjBimbkZC8aU5pwaxB0PoMAxzu7nBob1/UBOXqpsL88AN0n4PsKbWbDfZu+Dlog56PP/PSGVCLZidwRyO4C7WaVFoKqqq3pQ22a/zW3KTrSFSbOFeBPyT59v97sRxmBrBkVdG1CBadMM97K+9uMTBixTDsjY1u8bz1x8+hlVSZ8nDdJeagOfh2rikpHtvqU2yK87HVy2Mpw5FfEyu4yTge99fo3th0Bf7m1Jod7QBckIOg+HB/QFlrUOUB/IMTNTqPB+9RJ2AWXQkjZr5o1MeRDjopEpBij/L0PG+44bbytNZrk/Zx0XdnGAtrtlgH3qb4e4eqAheLPJVvdnxDcsUNfy6fMDpjHMddSnjp2kfueVzhtgWFkjMhAKhillveuoYmgJJx+KcrEikFS6bFcYQb/HNh+mI4vldiOc+fil29f/nwZiWyQxJzFNqdpljkITvH4NYNms3v4WqrQEVMbhfXBG7X458Dt2Wz8xihdgr6DviW2gDXZeWJXZPdc0Ta9qvWCIqhqOseOrQrx79W14gF0VXB5TSifsSurOdUr7WTYlNlnSw308IKu5qHbrFYp73B0D5NHvsiMlyi+JmNKuVI1VsKndPoCDVgXH4fptK9J/cCDnbdaqkmtoDZSVkq7/pzy4fkBfNWRzDy2OAENOgnBlgCUCuaddB5zLsbA+lfUQ/QNr3LlkkF4jINogdQMTjEDna1oggl0Ohx4D5KyVQY6B2MYEhn5tvT8YJBRE3ZmGsJlB0VYHy8jSMK0WfCR6H9/MRhxjj7yLvqPGJnI1BH0MI/RIf/SMvWZyuxigZJT+WP1tpqkMK19Bi3m0vbifL8xWzwDKtKWW4S4YYIr4k80yWa4YlAILwOCL20oeUQfAXJ+Yh3kAlv6Nna4T2A42PKZRBTGFFJs2UkxRWWhZM4XZD+ofhvO91+Vq3PCB4ZrVHgbo2CX+jpkoJboYTOeEyDu1CKYFioLQnMHpe3JdoF9Is7jfizHANdrspc+xAUkC9JyV3lrGzsrAYVjOpV+uxcNC3webIqxXUgNw71S4/nIpqt/WI263QuiDhRC4a0UyYpTzrI6wi3B4PLiB5lbCwFz98xXMUZapwaDByvl6VjNQfvLo0+cUCq0nG5S5SryjbqErBntcS5jVda00UWaa138RQGDfRlC4lKLyu0TZlrHLheQvFvSTREOQ7W2rA5nEdBix+nCpDWJhk8qJJX2UTStEYyzCwQXRe49FL2NmqswZWLOU8T4BKQHkzsljczBSscYIecRNuVNzoBLnZNfrZYCy7zbAOQ7/M97vFOAVXoIRfr3WmMBcJN3U8WZVhHI1Ft9uVNUq4VIWIlVqSpyGgktVapVDZI1AKuPlMHgiJvElM5cqf0zDlOkXv74yMAXo61O8BfMbQ/dNuclmSfUgq2CAZM+gQbMvjAawvqofnlPEuUL9m9rtkx7s1vX1Kg9R6+12IEan0o5eL3hR9wYUzfUL1YAbrLomjjppVbjl3AIopqDZoSV49L263ckBTAl3KfUl4fvyNa1NSIbyR1Le5wCspeGaTvFAvyazBM5ePwOMtQCQUiL4ciDyoLLHZqEPBzdyzHY7DKIGi9J2C6hcluDNIoKWXXQmMfrbVLmdZ2CzXAKu9/tTsb18vjwQxg4fm4L+d2FySgBsxDf3tASG46ipMUV5jvOi/p1D8iUGVMNrYIApr6Df5sg5k1ImlJk4uMoItq2s+3cMUmPf3tN2W3lfgEXG+nf6FCDLCN5FVaNWToMkikx0LV5hktiv2KUpZlve1vpnx9tzpuJZjGmMKcpnHZuAUuKdXsyCP/04liYjDee5RM4eh4Pc2lB/AUxAYaL6VcJsNAxQlQTuXbY0fBvY63S5WA9stuQYcUFqA713hFmjLl2WlcKt5i8WA22RP1vy3fS21872xE2FZDJsH4NAo409VuLqfD5bIq9k5ivSXRyf1Qzsiwqmgfg5R3JOZOfIWRg7+LBjnV7dWsH7Wqnyxt/j+zlP8l4XaXD06/r3kTpTaVlAWNfK/8ELyOmLMNW5zgLmqmoOfo5MtqzEH84e0hq8ntNtYbvVF1LVcny/+qi6pr6GMJdC0rbTgFIRUu6K6iVD4xYHCEZk9S6spK0qVbpKt5m/VtOU/etyVLrLCGQv9doJp8cn8WxySqSUcM7hUsJ5TZ8rNvlieAE9kziy8zhfS5FQyqrtMjK1O2sSJqsPr+qcKETXV7jljEi/RvlcqESNZ4+8E8AKPD194WbX0W7W5Jxp6kqZCHk9XL9eUCPao0+26EPEPGgwwySl5gqOPoWbohKfwSQ4Bkm3wciZvgy5BHv92oDsEr9nIPOisqTX1cw55wRY929nhdR1XwATMPrfqT/tbCkXG7QUZRDDXKXAwsQaNFKGaNsW5zxVCEWiKUNs1WbrJ4O8qLrVcU2qIiudKp5hs2W1WlHXFVWtdYNNpK8nMFVumcpaPZOtMqf3w0qmOQ8vnLSsZ07iVQTfM6vEMAoAVwLCW42pgWT57b3XN99mcs5kEi47Ukp0XwgGyLsbHMhEkYByc6Iv6c55kMwi6pW7ROwiOXZ0XUfdv72zgNg50b/upbf5W1KOOKcvpvaBYOViWQao6yJ1PVNTFAZJ6xeqMBxQEKmu2FGuUVcubzUcba6YxTQiek3kWfvVzwrTZ9cuYyA7Q1ciTbuek8iqd7iYiLEjZmWAGOli+wVgAGv98yli6ic8Fg8kr/bUBR2CqnzxGnKCTuf3V7ZiaOwGhJ4yKXq8qdbsFRHLiqWEimDTxlUAxYusqJqFVCA7lEC6ymhfPGoS3g3h6iD3FO2UBGvEVn8r+k9RytS8BqmM0fsoZQE2pxhg7P6Z51Rut3cY5IxLrscEMUVSSvz/iZqYF1qnge8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "Image (3, 128, 128)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "im.apply_tfms([rgb_offset(p=1.0, offsets=offsets)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "That's pretty extreme but let's see."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "data1 = get_data(128, True, 64, tfms=[flip_lr(p=0.5)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imagenet_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='1000' class='' max='1000', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [1000/1000 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "RunningStatistics(n=16384000, mean=[0.503,0.493,0.462], std=[0.255,0.252,0.264])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Non-normalized x's\n",
    "collect_stats(data1.train_ds.x[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def get_data_stats(data, total=1000):\n",
    "    stats = RunningStatistics()\n",
    "    for x,_ in progress_bar(data1.train_dl, total=1000//data.train_dl.dl.batch_size):\n",
    "        for b in range(x.shape[0]):\n",
    "            stats.update(x[b])\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='15' class='' max='15', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [15/15 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "RunningStatistics(n=15728640, mean=[0.0671,0.0353,-0.021], std=[1.09,1.07,1.1])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Normalised imagenette\n",
    "get_data_stats(data1, total=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "results = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "res = []\n",
    "for r in range(5):\n",
    "    lrn = cnn_learner(data1, models.resnet34, pretrained=True, metrics=[accuracy])\n",
    "    lrn.fit_one_cycle(5, 3e-3)\n",
    "    acc = lrn.recorder.metrics[-1][0].item()\n",
    "    res.append(acc)\n",
    "res = array(res)*100\n",
    "results['orig_imagenet_stats'] = res\n",
    "print(f\"{res.mean():2.2f} +- {res.std():0.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88.00 +- 0.87\n"
     ]
    }
   ],
   "source": [
    "print(f\"{res.mean():2.2f} +- {res.std():0.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## data2 - Offset data, no stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "off1 = rgb_offset(offsets=tensor(0.2,-0.2,0.2))\n",
    "data2 = get_data(128, True, 64, tfms=[flip_lr(p=0.5), off1], stats=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='15' class='' max='15', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [15/15 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "RunningStatistics(n=15728640, mean=[0.102,0.0473,-0.0172], std=[1.08,1.07,1.11])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats2 = get_data_stats(data2, total=1000)\n",
    "stats2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "### Show Batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {