Fast.ai p1v1: class 4

1.tabular.ipynb

2.lm-spanish.ipynb

3.nlp.ipynb

4.tabular-nlp.ipynb

5.full-deep-learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Full deep learning model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.tabular import *\n",
    "from fastai.text import *\n",
    "from fastai.metrics import accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_sz = 10000\n",
    "PATH = Path('~/data/').expanduser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_feather(PATH/'listings-df')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Input models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1. Tabular model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tab = df.drop('title', axis=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cont_cols = ['col1', 'col2', 'col3', 'col4', 'col5', 'col6',\n",
    "             'col7', 'col8', 'col9', 'col10', 'col11', 'col12'\n",
    "             'title_isnew_prob'] # real columns names were replaced\n",
    "cat_cols = sorted(list(set(df_tab.columns) - set(cont_cols) - {'condition'}))\n",
    "valid_idx = range(len(df)-valid_sz, len(df))\n",
    "procs = [FillMissing, Categorify, Normalize]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_tab = (TabularList.from_df(df_tab, cat_cols, cont_cols, procs=procs, path=PATH)\n",
    "            .split_by_idx(valid_idx)\n",
    "            .label_from_df(cols='condition')\n",
    "            .databunch())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_tab = tabular_learner(data_tab, layers=[64], ps=[0.5], emb_drop=0.05, metrics=accuracy)\n",
    "learn_tab.load('tabular-model');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TabularModel(\n",
       "  (embeds): ModuleList(\n",
       "    (0): Embedding(4, 3)\n",
       "    (1): Embedding(10492, 50)\n",
       "    (2): Embedding(3, 2)\n",
       "    (3): Embedding(8, 5)\n",
       "    (4): Embedding(1461, 50)\n",
       "    (5): Embedding(286, 50)\n",
       "    (6): Embedding(3481, 50)\n",
       "    (7): Embedding(304, 50)\n",
       "    (8): Embedding(570, 50)\n",
       "    (9): Embedding(30, 16)\n",
       "    (10): Embedding(26, 14)\n",
       "    (11): Embedding(300, 50)\n",
       "    (12): Embedding(33283, 50)\n",
       "    (13): Embedding(5, 3)\n",
       "    (14): Embedding(5, 3)\n",
       "    (15): Embedding(3, 2)\n",
       "    (16): Embedding(3, 2)\n",
       "  )\n",
       "  (emb_drop): Dropout(p=0.05)\n",
       "  (bn_cont): BatchNorm1d(12, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (layers): Sequential(\n",
       "    (0): Linear(in_features=462, out_features=64, bias=True)\n",
       "    (1): ReLU(inplace)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_tab.model.layers = learn_tab.model.layers[:-3]\n",
    "learn_tab.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2. NLP Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_nlp = df[['title', 'condition']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = pickle.load(open(PATH/'itos', 'rb'))\n",
    "data_nlp = TextClasDataBunch.from_df(PATH, df_nlp[:-valid_sz], df_nlp[-valid_sz:], \n",
    "                                     tokenizer=Tokenizer(lang='es'), \n",
    "                                     vocab=vocab, text_cols='title', label_cols='condition')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_nlp = text_classifier_learner(data_nlp, drop_mult=0.5)\n",
    "learn_nlp.load('nlp-final');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequentialRNN(\n",
       "  (0): MultiBatchRNNCore(\n",
       "    (encoder): Embedding(22847, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(22847, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1150)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1150, 1150)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1150, 400)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_nlp.model[-1].layers = learn_nlp.model[-1].layers[:-3] \n",
    "learn_nlp.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Concat model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConcatDataset(Dataset):\n",
    "    def __init__(self, x1, x2, y): self.x1,self.x2,self.y = x1,x2,y\n",
    "    def __len__(self): return len(self.y)\n",
    "    def __getitem__(self, i): return (self.x1[i], self.x2[i]), self.y[i]\n",
    "\n",
    "train_ds = ConcatDataset(data_tab.train_ds.x, data_nlp.train_ds.x, data_tab.train_ds.y)\n",
    "valid_ds = ConcatDataset(data_tab.valid_ds.x, data_nlp.valid_ds.x, data_tab.valid_ds.y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_collate(batch):    \n",
    "    x,y = list(zip(*batch))\n",
    "    x1,x2 = list(zip(*x))\n",
    "    x1 = to_data(x1)\n",
    "    x1 = list(zip(*x1))\n",
    "    x1 = torch.stack(x1[0]), torch.stack(x1[1])\n",
    "    x2, y = pad_collate(list(zip(x2, y)), pad_idx=1, pad_first=True)\n",
    "    return (x1, x2), y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64\n",
    "train_sampler = SortishSampler(data_nlp.train_ds.x, key=lambda t: len(data_nlp.train_ds[t][0].data), bs=bs//2)\n",
    "valid_sampler = SortSampler(data_nlp.valid_ds.x, key=lambda t: len(data_nlp.valid_ds[t][0].data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(train_ds, bs//2, sampler=train_sampler)\n",
    "valid_dl = DataLoader(valid_ds, bs, sampler=valid_sampler)\n",
    "data = DataBunch(train_dl, valid_dl, device=defaults.device, collate_fn=my_collate, path=PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape tabular batch (cats/cont): torch.Size([32, 17]) / torch.Size([32, 12])\n",
      "Shape nlp batch: torch.Size([42, 32])\n",
      "Shape dependent var: torch.Size([32])\n"
     ]
    }
   ],
   "source": [
    "(x1,x2),y = next(iter(data.train_dl))\n",
    "print(f'Shape tabular batch (cats/cont): {x1[0].shape} / {x1[1].shape}')\n",
    "print(f'Shape nlp batch: {x2.shape}')\n",
    "print(f'Shape dependent var: {y.shape}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConcatModel(nn.Module):\n",
    "    def __init__(self, mod_tab, mod_nlp, layers, drops): \n",
    "        super().__init__()\n",
    "        self.mod_tab = mod_tab\n",
    "        self.mod_nlp = mod_nlp\n",
    "        lst_layers = []\n",
    "        activs = [nn.ReLU(inplace=True),] * (len(layers)-2) + [None]\n",
    "        for n_in,n_out,p,actn in zip(layers[:-1], layers[1:], drops, activs):\n",
    "            lst_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn)\n",
    "        self.layers = nn.Sequential(*lst_layers)\n",
    "\n",
    "    def forward(self, *x):\n",
    "        x_tab = self.mod_tab(*x[0])\n",
    "        x_nlp = self.mod_nlp(x[1])[0]\n",
    "        x = torch.cat([x_tab, x_nlp], dim=1)\n",
    "        return self.layers(x)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ConcatModel(\n",
       "  (mod_tab): TabularModel(\n",
       "    (embeds): ModuleList(\n",
       "      (0): Embedding(4, 3)\n",
       "      (1): Embedding(10492, 50)\n",
       "      (2): Embedding(3, 2)\n",
       "      (3): Embedding(8, 5)\n",
       "      (4): Embedding(1461, 50)\n",
       "      (5): Embedding(286, 50)\n",
       "      (6): Embedding(3481, 50)\n",
       "      (7): Embedding(304, 50)\n",
       "      (8): Embedding(570, 50)\n",
       "      (9): Embedding(30, 16)\n",
       "      (10): Embedding(26, 14)\n",
       "      (11): Embedding(300, 50)\n",
       "      (12): Embedding(33283, 50)\n",
       "      (13): Embedding(5, 3)\n",
       "      (14): Embedding(5, 3)\n",
       "      (15): Embedding(3, 2)\n",
       "      (16): Embedding(3, 2)\n",
       "    )\n",
       "    (emb_drop): Dropout(p=0.05)\n",
       "    (bn_cont): BatchNorm1d(12, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (layers): Sequential(\n",
       "      (0): Linear(in_features=462, out_features=64, bias=True)\n",
       "      (1): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (mod_nlp): SequentialRNN(\n",
       "    (0): MultiBatchRNNCore(\n",
       "      (encoder): Embedding(22847, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(22847, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "    (1): PoolingLinearClassifier(\n",
       "      (layers): Sequential(\n",
       "        (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): Dropout(p=0.2)\n",
       "        (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "        (3): ReLU(inplace)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (layers): Sequential(\n",
       "    (0): BatchNorm1d(114, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (1): Dropout(p=0.8)\n",
       "    (2): Linear(in_features=114, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_layers = [64+50, 2]\n",
    "ps = [0.8]\n",
    "model = ConcatModel(learn_tab.model, learn_nlp.model, lin_layers, ps)\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_func = nn.CrossEntropyLoss()\n",
    "layer_groups = [nn.Sequential(*flatten_model(learn_nlp.layer_groups[0])),\n",
    "                nn.Sequential(*flatten_model(learn_nlp.layer_groups[1])),\n",
    "                nn.Sequential(*flatten_model(learn_nlp.layer_groups[2])),\n",
    "                nn.Sequential(*flatten_model(learn_nlp.layer_groups[3])),\n",
    "                nn.Sequential(*(flatten_model(learn_nlp.layer_groups[4]) + \n",
    "                                flatten_model(model.mod_tab) +\n",
    "                                flatten_model(model.layers)))] \n",
    "learn = Learner(data, model, loss_func=loss_func, metrics=accuracy, layer_groups=layer_groups)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.freeze()\n",
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl8VOW9x/HPLzsJCRASZA1JIAjIKmFT4IpV64q2bri07mup3qq1tt5qq711qa1rW8S61aVq8VqxVVFrURFBwio7YU/YQoAQkpD1uX9kwAECE5I5mUzyfb9e8+qcM8+Z83sazDfnPOc8x5xziIiIHE1EqAsQEZHmT2EhIiIBKSxERCQghYWIiASksBARkYAUFiIiEpDCQkREAlJYiIhIQAoLEREJKCrUBQRLSkqKS09PD3UZIiJhZd68eTucc6mB2rWYsEhPTycnJyfUZYiIhBUz21CfdjoNJSIiASksREQkIIWFiIgE5GlYmNmZZrbSzHLN7J46Pr/azArMbKHvdb3fZ9V+66d5WaeIiBydZwPcZhYJ/BE4HcgD5prZNOfcskOavumcm1THV5Q554Z4VZ+IiNSfl0cWI4Bc59xa51wF8AZwvof7ExERj3gZFt2ATX7Leb51h7rQzBab2VQz6+G3Ps7Mcsxstpld4GGdIiISgJdhYXWsO/QZru8B6c65QcAnwMt+n6U557KBy4EnzKzXYTswu9EXKDkFBQXBqvsgZRXVvLswn09XbEOPoBWR1srLm/LyAP8jhe7AZv8GzrlCv8XngEf8Ptvs+9+1ZjYDGAqsOWT7KcAUgOzs7KD+Jl+SX8Qbczfy7oLNFJdXATCuTyoPTDiB9JSEg9o656iucURF6uIyEWmZvAyLuUCWmWUA+cBEao8SDjCzLs65Lb7FCcBy3/oOQKlzrtzMUoCTgUc9rPWAfZXVXP9yDjNzdxAbFcHZA7twSXYPVm7dw+8/WsUZT3zOraf0YuLwNOau38kXqwv4YvUO9pRV8tRlQ/lOv+OaokwRkSblWVg456rMbBIwHYgEXnDOLTWzB4Ac59w04DYzmwBUATuBq32b9wOeNbMaak+VPVzHVVSe+NW0pczM3cHPz+rLxBFptGsTDcDoXh05e2AXfvOv5TzxyWqe+GQ1AElxUYzJSmFDYSk3vTKPpy4bytkDuzRFqQHtP21mVtcZQRGR+rOWch4+OzvbNXZuqLfmbuLutxfzo/G9+Ol3+x6x3aw1O1icV8TIjGQGdW9PZISxZ18l17w4lwUbd/H7SwbzvaHdASgpr+KdBflMW7iZK0f3ZMLgro2qsb6W5Bdx62vzObl3Cg99f2CT7FNEwo+ZzfONDx+9ncKi1pL8Ii788yyy0zvw12tHEhlx7H+Nl5RXcf3LOcxeV8jPzuzLtj37mJqTR3F5FUlxURSXV/HIhYO4JLvHYdt+taYQh+OkXikN7sN+7y7M5+6pi3FARVUNL14znPHHd2r094pIy1PfsNCILFBUWsktr80jOSGGpyYObVBQACTERvHiNcMZl5XKwx+s4NXZGzi1XyfevuUk5vziNMb0TuHuqYt5bc63kzzuLKngjjcXctlzs7n8uTn8bvoKqmsaFuDVNY6H3l/O7W8sZHD39sy46xR6d2rLvf/3DXt9g/QiIg3R6o8samoc1708l5m5O3jrptEMTevQ6FrKq6qZvnQbozM7kpoYe2D9vspqbn1tPp+u2M795/UnOSGGX7+3jD1lldxySi8Kist5Y+4mxvRO4cmJQ+jYNvYoezl8nze9Mo8ZKwv4waie/PLc/sRERTBvwy4umjyLK0f25MELBjS6byLSstT3yKLFPM+iodYXljB/427uO7d/UIICIDYqss6xibjoSCZfOYwf/20+v36vdrx+SI/2PHzhQPp2TgJgaFp7fvnuUs57eibPXHEiJ9ajJucc9/1jKTNWFvCbCwZw5aieBz4b1rMD15yUwQtfruO8wV0ZkZEclD6KSOvS6o8sAHbsLadjQkyTXTVUWV3DYx+tpGu7Nlw5qudhp72+ySviltfmkberjLFZKVw1Op3xfTsd8fTYy7PWc/+0pfz41N7cecbxh31eWlHFGY9/TnRkBB/cPpa46EhP+iUi4UcD3GGuqKySV2dv4JWvNrB1zz7SkuP54eieXDq8B4lx0QfazVqzgx88/zXjj09lyg+yiThCoMxcvYMrn5/D8PQOdEqMo6K6hsrqGtrGRjEuK5VT+qbSKTGuqbonIs2EwqKFqKyu4aOl23hp1jrmrt9FYmwUl49K45qTMqisrmHCMzPp2DaWd2496aAQqcvjH6/inQX5xERFEB0ZQUyksW1POVv37ANgYLd2nDmgMzeMzSQmStc+iLQGCosWaNGm3Uz5Yi0ffLOFyAijQ3wM+yqreXfSGDIOmYKkvpxzLN9SzH9Wbuffy7cxf+NuTurVkck/GEZSgPARkfCnsGjBNhaW8sKX65i+dCuPXDiIcX1Sg/bd/zc/j7unLqZ3p7a8eM1wurRrE7TvFpHmR2EhDTZz9Q5ufnUeiXFRvHTNCI7vnBjqkkTEI7opTxpsTFYKb940iuoax0WTZ7EkvyjUJYlIiCkspE4ndG3HOz86mYSYKG59bT5FZZX13ramgXegi0jz1epvypMj69a+DX+8YiiXPjubn/59Ec/+YNhR70VZkl/Evf9YwqJNu2kbG1X7iotiQNckHrloELFRur9DJFwpLOSohvVM5p6z+vKbfy3n+ZnruH5s5mFtyiqqeeKTVfxl5jo6xMdwyym9KK+soaS8il2lFfxj4Wbax8fwqwknhKAHIhIMCgsJ6LoxGcxdv5OHP1jB0LT2DOtZO2VIUVklX6wu4JEPV7BpZxkTh/fg52f1o138wZfcPvjPZTw/cx0jM5I5K8CzPkorqpizbif9uyRxXJJuEhRpLnQ1lNRLUVkl5z09k4qqGs4a2Jk5a3eyfOsenIPMlAR++/2BjMrsWOe2FVU1XPzsV6zdvpd/3TaWtI7xB32+r7KaGSu3897iLXy6fDtlldXERkVw1Unp3PJfveiQEFPvOssqqpk6bxOz1hTyX31SOWdQl4A3K4q0Zrp0VoJuSX4RF02ehXO1ExSOzOjIyMxkTkzrEPCO7007SznnqS/o2TGBqbeMJjYqkpVbi3l19gb+sSCf4vIqkhNiOGtAZ07t24l/fbOFdxbk0zYmihvGZXLhsO4clxh7xOecFxSX89ev1vPq7A3sKq0kOSGGnSUVxEVHcNaALlw0rDujMzsecToUkdZKYSGe2FlSQUJsZIMGqz9aupUbX5nHGf2PY1dpBXPX7yImKoJzBnbh+yd2Y3Rmx4PCYNW2Yh6bvpKPlm0DIDLC6JwUR9f2cSTERlFaUU1ZRTWlFVVs2llGZU0Np/c7jhvGZZLdswOL8or4e84mpi3aTPG+KtKS47l8ZBoXD+t+TNO/i7RkCgtplvaPX/TsGM/lI9K4OLsHyQFOMy3JL2JxXhH5u0vZvHsf+bvKKK2sIj4miviYSOJjIunarg2Xj0wjM7XtYdvvq6xm+tKtvD5nI3PW7SQ60jhzQBcGd29HUpto2reJpl2baAZ0a0dCrIbxpHVRWEizVFPjWLG1mL6dE0NySih3ezGvzdnI2/Py2LPv4KcHpiXH89I1w+sMHH/OObbtKWdDYQmJcdF0SIimQ3yMpn6XsKSwEDmKmhpHcXkVe8oqKSqrZOPOUv7nH0twzvGXq4YzrOfBD536Jq+I95dsYUl+Ecs276GwpOKw70yIiWRERjLf6Xccp/U7js7tdDWXNH8KC5FjtH5HCVe/+DVbivbx1GVDOaP/cczM3cHkz9bwZW4h0ZFGVqdETuiaxAldk8hMbUtpRRW7SivZVVrBlt37+Hx1ARsKS4HaKd9vOaUXZwe4XFgklBQWIg1QuLec617OYVHebnqltiV3+146JcZy7ZgMLh+ZFnDaduccudv38sny7by7MJ8VW4u58MTu/Pr8E2ir8RBphhQWIg1UVlHNXX9fRO72vVw7Jp0LhnZr0NVfldU1PP3v1Tzzn1y6d4jn8UuHHHZ6SyTUFBYizcTc9Tv5yZsL2VK0jxvHZTJpfG9ddSXNhqYoF2kmhqcn8/7tY/ne0G78ecYaxj82g6nz8jQ7r4QVHVmINKH5G3fx6/eWsWjTbgZ1b8c1J6cTFxWJmREZYSTERnJCl3aHza8l4hWdhhJppmpqHO8uyufhD1awbU95nW0yUxIY1L0dw9KTuXhYd93DIZ5RWIg0c/sqq9lQWEqNc9Q4h3O106l8k1/Eok27WbhpN9uLy+mVmsBjFw9maJoGxyX46hsWGmUTCZG46Mg6n28+rk/qgfefryrgnrcXc+GfZ3HDuEx+clofHWVISGiAW6QZG9cnlek/Gcelw3vw7GdrOeepL8jdXhzqsqQVUliINHOJcdE89P1B/PXaERSVVTFxymxWblVgSNNSWIiEiXF9UnnzplFEmHHZc7NZvmVPqEuSVkRhIRJGeqW25c2bRhMTGcHlz81m6eaiUJckrYTCQiTMZKQk8OZNo2gTHcnlz83hTzNyeWdBHrNyd5C7fS9V1TWhLlFaIF0NJRKGenZM4M2bRnPNS3N59MOVB33WKzWBP10xrM4rrUQaSvdZiIS5kvIqtu3Zd+CBTI99tIq95ZU8cP4ALh7WHTM9d1yOrFnMDWVmZ5rZSjPLNbN76vj8ajMrMLOFvtf1fp9dZWarfa+rvKxTJJwlxEaRmdqW0b06MnFEGu/fPoYT0zpw99TF3PnWIkrKqwJ/iUgAnoWFmUUCfwTOAvoDl5lZ/zqavumcG+J7/cW3bTJwPzASGAHcb2a6fVWkHjolxvHKdSP5yWl9eGdhPhf+eRa7Sw9/sp/IsfDyyGIEkOucW+ucqwDeAM6v57bfBT52zu10zu0CPgbO9KhOkRYnMsK4/bQsXrx6OGsLSrjmpbk6wpBG8TIsugGb/JbzfOsOdaGZLTazqWbW41i2NbMbzSzHzHIKCgqCVbdIi3HK8Z14+vKhLNq0m5tfnUd5VXWoS5Iw5WVY1DWqduho+ntAunNuEPAJ8PIxbItzbopzLts5l52amlrHJiLy3RM68+hFg/li9Q5u/9tCXVorDeJlWOQBPfyWuwOb/Rs45wqdc/vnaH4OGFbfbUWk/i4a1p37zu3Ph0u3cu87S2gpV0FK0/EyLOYCWWaWYWYxwERgmn8DM+vitzgBWO57Px04w8w6+Aa2z/CtE5EGunZMBj8+tTdv5mzi/+bnh7ocCTOe3ZTnnKsys0nU/pKPBF5wzi01sweAHOfcNOA2M5sAVAE7gat92+40swepDRyAB5xzO72qVaS1+O/T+jBn7U5+NW0po3p1pFv7NqEuScKEbsoTaWU2FpZy1pOfM7hHe169biQREbpprzVrFjfliUjzk9Yxnv85tz+z1hTy16/Wh7ocCRMKC5FWaOLwHow/PpWHPlhB7va9oS5HwoDCQqQVMjMeuXAQbWIiufMtXU4rgSksRFqpTklxPHj+ABblFfGXmetCXY40cwoLkVbs3EFdOL3/cTz+8So2FJaEuhxpxhQWIq2YmfHg+QOIjozgF+98o5v15IgUFiKtXOd2cfzsrL58mVvI27pZT45AYSEiXDEijeyeHfjNv5axY2954A2k1VFYiAgREcZD3x9ISXkVD/5zWajLkWZIYSEiAGQdl8itp/Tm3YWb+Wjp1lCXI82MwkJEDrh1fC8GdmvHnW8tYm2BbtaTbyksROSA2KhI/nzliURHRXDTK/PYq6friY/CQkQO0r1DPM9cNpQ1BXu5e+oiXU4rgMJCROpwUu8U7jmrL+9/s5XJn60NdTnSDCgsRKRON4zN5JxBXfjd9BV8sVrPuG/tFBYiUicz49ELB5HVKZFJry9gY2FpqEuSEFJYiMgRJcRGMeWHwwC48ZUcSjTg3WopLETkqHp2TODpy4ayalsxP9WAd6ulsBCRgMb1ST0w4P2nGWtCXY6EgMJCROrlhrGZTBjclcc+Wskny7aFuhxpYgoLEamX/U/XO6FrEje/Oo/X52wMdUnShBQWIlJvbWIief2GUYzJSuEX73zD/e8u0SNZWwmFhYgck6S4aJ6/ajg3jM3g5a82cPWLcykqrQx1WeIxhYWIHLPICOPec/rz6EWDmLOukEue/Yp9ldWhLks8pLAQkQa7JLsHU36QzcptxTw2fWWoyxEPKSxEpFHG9+3ElaPSeP7LdcxZWxjqcsQjCgsRabSfn9WPtOR47pq6SHd5t1AKCxFptITYKB67eDB5u8r47fvLQ12OeEBhISJBMTw9mRvGZvLanI18tkqz1LY0CgsRCZo7Tu9D705t+dnUxRTv0+W0LYnCQkSCJi46kscuHsy24n08/vHqUJcjQaSwEJGgGtKjPZePSOOlWetYurko1OVIkCgsRCTo7v5uXzrEx/DLfyyhpkZTmrcECgsRCbp28dH8/Ox+zN+4m7/P2xTqciQIFBYi4okLT+zGiPRkHvpgBTtLKkJdjjSSwkJEPGFmPHjBAIr3VfHohytCXY40kqdhYWZnmtlKM8s1s3uO0u4iM3Nmlu1bTjezMjNb6HtN9rJOEfHG8Z0TuW5MBm/M3cTCTbtDXY40gmdhYWaRwB+Bs4D+wGVm1r+OdonAbcCcQz5a45wb4nvd7FWdIuKt276TRUrbWH77r+V6fncY8/LIYgSQ65xb65yrAN4Azq+j3YPAo8A+D2sRkRBpGxvFf5+Wxdfrd/KxHscatrwMi26A/2UQeb51B5jZUKCHc+6fdWyfYWYLzOwzMxvrYZ0i4rFLh/cgMzWBhz9cQaWerBeW6hUWZtbLzGJ9708xs9vMrH2gzepYd+AY1MwigMeBO+totwVIc84NBe4AXjezpDrqutHMcswsp6BAc9GINFfRkRHcc2Zf1haU8MZcXUobjup7ZPE2UG1mvYHngQzg9QDb5AE9/Ja7A5v9lhOBAcAMM1sPjAKmmVm2c67cOVcI4JybB6wB+hy6A+fcFOdctnMuOzU1tZ5dEZFQOL3/cYxIT+bJT1axV9OYh536hkWNc64K+B7whHPuJ0CXANvMBbLMLMPMYoCJwLT9HzrnipxzKc65dOdcOjAbmOCcyzGzVN8AOWaWCWQBa4+pZyLSrJgZvzinHzv2VvDsZ2tCXY4co/qGRaWZXQZcBewfX4g+2ga+cJkETAeWA28555aa2QNmNiHA/sYBi81sETAVuNk5t7OetYpIMzWkR3vOHdSF575Yy9YiXdMSTqw+l7L5Lnm9GfjKOfc3M8sALnXOPex1gfWVnZ3tcnJyQl2GiASwsbCUU38/gx+OTue+8w67ml6amJnNc85lB2pXryML59wy59xtvqDoACQ2p6AQkfCR1jGecwd14c25Gykq0zMvwkV9r4aaYWZJZpYMLAJeNLM/eFuaiLRU14/NpKSimr99vTHUpUg91XfMop1zbg/wfeBF59ww4DTvyhKRlmxAt3ac1KsjL325nooq3XcRDuobFlFm1gW4hG8HuEVEGuyGcZls3bOPfy7eHLixhFx9w+IBaq9qWuOcm+u7nFXPTBSRBjulTypZndry3BfrNGdUGKjvAPffnXODnHO3+JbXOucu9LY0EWnJzIwbxmayfMsevswtDHU5EkB9B7i7m9k7ZrbdzLaZ2dtm1t3r4kSkZTt/aFdS2sby3Be657a5q+9pqBepvfu6K7WTAb7nWyci0mCxUZFcfVJPPltVwMqtxaEuR46ivmGR6px70TlX5Xu9BGgyJhFptCtG9qRNdCRPfLIq1KXIUdQ3LHaY2ZVmFul7XQnoJKOINFqHhBgmndqbD5ZsZcbK7aEuR46gvmFxLbWXzW6ldvrwi4BrvCpKRFqX68dmkJmSwP3TlrKvsjrU5Ugd6ns11Ebn3ATnXKpzrpNz7gJqb9ATEWm02KhIHjh/ABsKS3n2Mw12N0eNeVLeHUGrQkRavTFZKZw3uCt/nJHLhsKSUJcjh2hMWNT1JDwRkQb7n3P6ERMZwX3vLtWNes1MY8JCP0kRCarjkuK44/Q+fLaqgA+XbA11OeLnqGFhZsVmtqeOVzG191yIiATVD0f3pH+XJH729mJWbdO9F83FUcPCOZfonEuq45XonItqqiJFpPWIiozg2R8MIy46kqte+JrNu8tCXZLQuNNQIiKe6JEcz8vXjmDvvip++MLX7C6tCHVJrZ7CQkSapX5dkpjyw2w2FpZy3cs5lFXo/otQUliISLM1uldHnpg4hPkbd3HbGwuoqdF1NaGisBCRZu3sgV24/9z+fLxsG3/4WPNHhYoGqUWk2bvqpHRWbC3mmf/k0r9rEmcP7BLqklodHVmISLNnZvz6/BM4Ma09d761iOVb9oS6pFZHYSEiYSE2KpLJVw4jqU0UN76Sw64SXSHVlBQWIhI2OiXFMfnKYWwrKmfS3+ZrwLsJKSxEJKwMTevA/RP682VuIdOXakqQpqKwEJGwM3F4GpmpCTz579U6umgiCgsRCTuREcaPT+3Niq3FfLRsW6jLaRUUFiISls4b1JXMFB1dNBWFhYiEpajICCad2pvlW/bw8XIdXXhNYSEiYWvC4K6kd4znyU9W62FJHlNYiEjYqj26yGLZlj18rLELTyksRCSsXTCkKz07xvPkv3V04SWFhYiEtajICCaN783SzXt4/xvdd+EVhYWIhL3vn9idfl2S+O37y/XcC48oLEQk7EVGGL86rz/5u8uY/NmaUJfTIiksRKRFGJnZkfMGd2XyZ2vYtLM01OW0OJ6GhZmdaWYrzSzXzO45SruLzMyZWbbfup/7tltpZt/1sk4RaRl+cXZfIsz4338tD3UpTWbyZ2v4w8erPB/c9ywszCwS+CNwFtAfuMzM+tfRLhG4DZjjt64/MBE4ATgT+JPv+0REjqhLuzZMOrU3Hy7dyszVO0Jdjud2llTwzKe55G4vxsw83ZeXRxYjgFzn3FrnXAXwBnB+He0eBB4F9vmtOx94wzlX7pxbB+T6vk9E5KiuG5NBWnI8v35vKZXVNaEux1PPfraG0ooq7ji9j+f78jIsugGb/JbzfOsOMLOhQA/n3D+PdVsRkbrERUfyy3P7s3r7Xp78ZHWoy/HM9j37ePmr9VwwpBu9OyV6vj8vw6KuY6IDJ9XMLAJ4HLjzWLf1+44bzSzHzHIKCgoaXKiItCyn9evEJdndeeY/uby7MD/U5XjiTzPWUFntuP20rCbZn5dhkQf08FvuDmz2W04EBgAzzGw9MAqY5hvkDrQtAM65Kc65bOdcdmpqapDLF5FwZWb85oKBjMhI5qdTFzNvw65QlxRU+bvLeH3ORi7J7k7PjglNsk8vw2IukGVmGWYWQ+2A9bT9HzrnipxzKc65dOdcOjAbmOCcy/G1m2hmsWaWAWQBX3tYq4i0MDFREUy+chidk+K46ZUc8na1nMtpn/m09vTapFOb5qgCPAwL51wVMAmYDiwH3nLOLTWzB8xsQoBtlwJvAcuAD4EfOed0W6aIHJPkhBheuDqb8qoarn85h73lVaEuqdHW7yjhrZw8Lh+ZRrf2bZpsv9ZSJt7Kzs52OTk5oS5DRJqhL1YXcPWLc7l0eA9++72BoS6nUe54cyHvL9nC53ePp1NiXKO/z8zmOeeyA7XTHdwi0uKNzUpl4vAeTM3JY9uefYE3aKacc3y8bBsXDOkWlKA4FgoLEWkVbhrXi6qaGp6fuS7UpTRYYUkFxeVVHN/Z+0tlD6WwEJFWIa1jPOcO6sprszdQVFoZ6nIaZP2OEgDSU5rmCih/CgsRaTVuOaUXJRXVvDJ7fahLaZB1vrDIaKLLZf0pLESk1ejXJYnxx6fywpfrw/K5FxsKS4mMMLp1aLqroPZTWIhIq3Lr+N7sLKngrZxNgRs3M+sKS+jRoQ3RkU3/q1thISKtyvD0ZLJ7dmDK52vDbqLB9TtKQjJeAQoLEWmFbh3fi/zdZby36LBZhJot51xtWIRgvAIUFiLSCo0/vhN9Oyfy2PSV7CqpCHU59bJjbwUlFdWkd4wPyf4VFiLS6pgZj140iB17K7jjrYXU1DT/mSzWF4busllQWIhIKzWoe3t+eW4//rOygMmfrwl1OQEduGxWYSEi0rSuHNWT8wZ35bHpK5m9tjDU5RzVhsISoiKsSScP9KewEJFWy8x46PsDSe+YwG1/W0BBcXmoSzqi9TtK6ZEcT1QILpsFhYWItHJtY6P405UnUlRWyR1vLaS5zsS9bkdJyAa3QWEhIkLfzkn87My+fLF6B7PWNL/TUc45NhSWNNlT8eqisBARAS4fmcZxSbE89e/VoS7lMAV7yympqA7Z4DYoLEREAIiLjuSmcb2Ys24nc5rZYPf6HbWPhA3VZbOgsBAROeCyEWmktI3l6U9zQ13KQQ7cY6ExCxGR0GsTE8mN4zKYmbuDeRt2hbqcA9bvCO1ls6CwEBE5yBUje5KcEMPTnzafsYv1hSWkhfCyWVBYiIgcJCE2iuvGZDBjZQGL83aHuhygdsyiZwhPQYHCQkTkMD8c3ZN2baJ56t+hH7twzrG+MHRTk++nsBAROURiXDTXjcngk+Xb+HjZtpDWUlBcTmmIL5sFhYWISJ1uHJfJwG7tuOOthWzwXY0UCvsnEAzVcyz2U1iIiNQhLjqSP11xIhFm3PzqfPZVhuaZ3RsKffdYKCxERJqnHsnxPHHpEJZv2cMv/7EkJDWsKywhOtLo2j4uJPvfT2EhInIU4/t24rZTe/P3eXm8OXdjk+9//Y6SkM42u5/CQkQkgNtP68PYrBR++e5SluQXNem+1xeWhvwUFCgsREQCiowwnpw4lOT4GCa9Pp+95VWe79M5R+72YjYUligsRETCRXJCDE9OHMLGnaXc+843nj33Ys7aQn41bSnjfvcfTvvD55RWVJOd3sGTfR2LqFAXICISLkZmduQnp/Xh9x+v4uReKVwyvEdQv3/+xl1cOmU2sVERjOmdws3/1Yvxx3eiawjnhNpPYSEicgxuHd+b2esKuW/aEoaktafPcYlB++7PVxUQYfDVz79DckJM0L43GHQaSkTkGERGGI9fOoS2sVH86LX5lFUE7/6L2WsL6d81qdkFBSgsRESOWafEOJ64dCi5BXt5MkjreqidAAAMIUlEQVRP1ttXWc2CjbsZmdExKN8XbAoLEZEGGJOVwgVDuvHil+vYUlTW6O9btGk35VU1jMpUWIiItCh3nN4H5+Dxj1c1+rvmrNuJGYxITw5CZcGnsBARaaAeyfH8cHRPps7LY9W24kZ91+y1hfTrnES7+OggVRdcnoaFmZ1pZivNLNfM7qnj85vN7BszW2hmM82sv299upmV+dYvNLPJXtYpItJQPxrfm4TYKB75YEWDv6O8qpr5G3cxMrN5HlWAh2FhZpHAH4GzgP7AZfvDwM/rzrmBzrkhwKPAH/w+W+OcG+J73exVnSIijdEhIYZbTunFv1dsZ87awgZ9x+K8IvZVNt/xCvD2yGIEkOucW+ucqwDeAM73b+Cc2+O3mAB4c0ukiIiHrj05g85JcTz84YoG3dm9P2Sa63gFeBsW3YBNfst5vnUHMbMfmdkaao8sbvP7KMPMFpjZZ2Y2tq4dmNmNZpZjZjkFBQXBrF1EpN7ioiP5yelZLNi4mw+XbD3m7Wev3Unfzol0aIb3V+znZVhYHesOi1zn3B+dc72AnwH/41u9BUhzzg0F7gBeN7OkOrad4pzLds5lp6amBrF0EZFjc+GJ3cnq1JbHPlpJVXVNvberqKph3oZdzfoUFHgbFnmA/8Qp3YHNR2n/BnABgHOu3DlX6Hs/D1gD9PGoThGRRouKjODOM/qwpqCEdxbk13u7b/J3U1ZZzciM5nsKCrwNi7lAlpllmFkMMBGY5t/AzLL8Fs8BVvvWp/oGyDGzTCALWOthrSIijfbdEzozsFs7nvhkNeVV9ZsGZPbanQCMaK1h4ZyrAiYB04HlwFvOuaVm9oCZTfA1m2RmS81sIbWnm67yrR8HLDazRcBU4Gbn3E6vahURCQYz467vHk/+7jLenLsp8AbU3l/R57i2dGwb63F1jePprLPOufeB9w9Zd5/f+9uPsN3bwNte1iYi4oVxWSmMyEjm6U9zuXhYD9rERB6xbWV17XjFRcO6N2GFDaM7uEVEgsjM+Ol3j6eguJyXv1p/1Lbf5BdRWlHdbCcP9KewEBEJsuHpyZxyfCp/nrGGPfsqj9juwyVbiYywZn3n9n4KCxERD9x1xvEUlVXy3Od1X5tTWlHFG19v5MwBnUlp5uMVoLAQEfHEgG7tOGdgF56fuY6C4vLDPn97fj579lVx7cnpTV9cAygsREQ8cucZfSivquHpTw9+QFJNjePFL9cxqHs7TkzrEKLqjo3CQkTEI5mpbZk4vAevz9nI+h0lB9Z/vrqAtQUlXHtyBmZ1TXbR/CgsREQ8dPt3soiOjOCxj1YeWPfil+vplBjL2QO7hLCyY6OwEBHxUKekOK4fm8E/F29hcd5ucrfv5bNVBVw5qicxUeHzKzh8KhURCVM3jsukQ3w0j3y4gpdmrSMmKoLLR6aFuqxjorAQEfFYYlw0Pz41iy9zC3nj602cP7hrWFwu609hISLSBK4YlUb3Dm2oqnFcc3JGqMs5Zp7ODSUiIrVioyJ57OLBLNi4m/5dD3s8T7OnsBARaSKjMjs2+4ccHYlOQ4mISEAKCxERCUhhISIiASksREQkIIWFiIgEpLAQEZGAFBYiIhKQwkJERAIy51yoawgKMysANhyyuh1QdIzrAr1PAXY0otS69l/fNvVdHy79OdJn6k/L6I//sv/6xvSpqftz6HIo+nO0dsHoT3vnXGrACpxzLfYFTDnWdYHeAznBrqm+beq7Plz6U9+fkfoTnv05pB/+bRrcp6buz1F+Lk3Wn6O1C1Z/6vNq6aeh3mvAuvq8b4z6fM+R2tR3fbj050ifqT8toz/+y+Han0OXQ9Gfo7ULVn8CajGnoZqKmeU457JDXUewqD/NW0vrD7S8PrW0/hxJSz+y8MKUUBcQZOpP89bS+gMtr08trT910pGFiIgEpCMLEREJqFWHhZm9YGbbzWxJA7YdZmbfmFmumT1lZub32Y/NbKWZLTWzR4Nb9VFrCnp/zOxXZpZvZgt9r7ODX/kRa/Lk5+P7/C4zc2aWEryKA9bkxc/nQTNb7PvZfGRmXYNf+RFr8qI/vzOzFb4+vWNm7YNf+RFr8qI/F/t+D9SYWXiPazT0kq+W8ALGAScCSxqw7dfAaMCAD4CzfOvHA58Asb7lTmHen18Bd7WUn4/vsx7AdGrvy0kJ5/4ASX5tbgMmh3l/zgCifO8fAR4J8/70A44HZgDZTdUXL16t+sjCOfc5sNN/nZn1MrMPzWyemX1hZn0P3c7MulD7H+lXrvZfxF+BC3wf3wI87Jwr9+1ju7e9+JZH/QkZD/vzOHA30KQDdl70xzm3x69pAk3YJ4/685FzrsrXdDbQ3dtefMuj/ix3zq1sivq91qrD4gimAD92zg0D7gL+VEebbkCe33Kebx1AH2Csmc0xs8/MbLin1QbW2P4ATPKdFnjBzDp4V2q9NKo/ZjYByHfOLfK60Hpq9M/HzP7XzDYBVwD3eVhrfQTj39t+11L7V3ooBbM/YU3P4PZjZm2Bk4C/+53ijq2raR3r9v9FFwV0AEYBw4G3zCzT9xdHkwpSf/4MPOhbfhD4PbX/ETe5xvbHzOKBe6k91RFyQfr54Jy7F7jXzH4OTALuD3Kp9RKs/vi+616gCngtmDUei2D2pyVQWBwsAtjtnBviv9LMIoF5vsVp1P4C9T887g5s9r3PA/7PFw5fm1kNtXPHFHhZ+BE0uj/OuW1+2z0H/NPLggNobH96ARnAIt9//N2B+WY2wjm31ePa6xKMf2/+Xgf+RYjCgiD1x8yuAs4FvhOKP7L8BPvnE95CPWgS6heQjt+AFjALuNj33oDBR9huLrVHD/sHtM72rb8ZeMD3vg+wCd/9LGHany5+bX4CvBHOP59D2qynCQe4Pfr5ZPm1+TEwNcz7cyawDEhtyn54/e+NFjDAHfICQtp5+BuwBaik9ojgOmr/8vwQWOT7R3vfEbbNBpYAa4Bn9gcCEAO86vtsPnBqmPfnFeAbYDG1f0V1Cef+HNKmScPCo5/P2771i6md56dbmPcnl9o/sBb6Xk15dZcX/fme77vKgW3A9KbqT7BfuoNbREQC0tVQIiISkMJCREQCUliIiEhACgsREQlIYSEiIgEpLKRFM7O9Tby/v5hZ/yB9V7VvNtklZvZeoBlYzay9md0ajH2LHEqXzkqLZmZ7nXNtg/h9Ue7bie485V+7mb0MrHLO/e9R2qcD/3TODWiK+qR10ZGFtDpmlmpmb5vZXN/rZN/6EWY2y8wW+P73eN/6q83s72b2HvCRmZ1iZjPMbKrv2Quv+T2/YMb+5xaY2V7fJH+LzGy2mR3nW9/LtzzXzB6o59HPV3w7GWJbM/u3mc232mconO9r8zDQy3c08jtf25/69rPYzH4dxP8bpZVRWEhr9CTwuHNuOHAh8Bff+hXAOOfcUGpnb/2t3zajgaucc6f6locC/w30BzKBk+vYTwIw2zk3GPgcuMFv/0/69h9wDiHfXETfofYOeoB9wPeccydS+/yU3/vC6h5gjXNuiHPup2Z2BpAFjACGAMPMbFyg/YnURRMJSmt0GtDfbybRJDNLBNoBL5tZFrWzhkb7bfOxc87/WQdfO+fyAMxsIbVzCs08ZD8VfDvx4jzgdN/70Xz7fI3XgceOUGcbv++eB3zsW2/Ab32/+GuoPeI4ro7tz/C9FviW21IbHp8fYX8iR6SwkNYoAhjtnCvzX2lmTwP/cc59z3f+f4bfxyWHfEe53/tq6v5vqdJ9Oyh4pDZHU+acG2Jm7agNnR8BT1H73IpUYJhzrtLM1gNxdWxvwEPOuWePcb8ih9FpKGmNPqL2uQ8AmNn+KajbAfm+91d7uP/Z1J7+ApgYqLFzrojaR6beZWbR1Na53RcU44GevqbFQKLfptOBa33PZcDMuplZpyD1QVoZhYW0dPFmluf3uoPaX7zZvkHfZdROKw/wKPCQmX0JRHpY038Dd5jZ10AXoCjQBs65BdTOfDqR2gcCZZtZDrVHGSt8bQqBL32X2v7OOfcRtae5vjKzb4CpHBwmIvWmS2dFmpjviX1lzjlnZhOBy5xz5wfaTiSUNGYh0vSGAc/4rmDaTYgeUytyLHRkISIiAWnMQkREAlJYiIhIQAoLEREJSGEhIiIBKSxERCQghYWIiAT0/xNH9YCbcy2eAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:37\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.106572    0.248390    0.920200  (00:37)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:40\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.086336    0.256554    0.919800  (00:40)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, slice(5e-3/(2.6**4), 5e-3), moms=(0.8, 0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 01:03\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.097170    0.257217    0.919500  (01:03)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-3)\n",
    "learn.fit_one_cycle(1, slice(2e-3/(2.6**4), 2e-3), moms=(0.8, 0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 07:04\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.080045    0.260310    0.920200  (01:24)\n",
      "2      0.075644    0.249944    0.922800  (01:26)\n",
      "3      0.071381    0.271557    0.920900  (01:26)\n",
      "4      0.078788    0.290130    0.919600  (01:24)\n",
      "5      0.088786    0.268973    0.921800  (01:23)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(5, slice(5e-4/(2.6**4), 5e-4), moms=(0.8, 0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 06:51\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      0.077082    0.248748    0.924100  (01:21)\n",
      "2      0.081846    0.249953    0.923700  (01:20)\n",
      "3      0.088959    0.254498    0.920200  (01:23)\n",
      "4      0.056842    0.249644    0.922800  (01:21)\n",
      "5      0.067153    0.244735    0.922900  (01:24)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(5, slice(5e-4/(2.6**4), 5e-4), moms=(0.8, 0.7), wd=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:fastai]",
   "language": "python",
   "name": "conda-env-fastai-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

make gist public

Hello! You've done nice job and I've got a question. When you finish training this model, how can you predict one example? It's not working with .predict(example).
I've done this with two AWD_LSTM networks, but in the end I've met an issue with this error while making prediction:
AttributeError: 'ConcatDataset' object has no attribute 'set_item'

Best regards

Hello! You've done nice job and I've got a question. When you finish training this model, how can you predict one example? It's not working with .predict(example).
I've done this with two AWD_LSTM networks, but in the end I've met an issue with this error while making prediction:
AttributeError: 'ConcatDataset' object has no attribute 'set_item'

Best regards

Same problem here