From af33f347938a6ea6c43a6c764c8745d9889cae6f Mon Sep 17 00:00:00 2001 From: glcanvas Date: Wed, 30 Oct 2019 23:45:55 +0300 Subject: [PATCH 01/10] simple network from tutorial --- .idea/.gitignore | 2 + .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 7 + .idea/ml-diplom.iml | 16 + .idea/modules.xml | 8 + .idea/other.xml | 6 + .idea/vcs.xml | 6 + README.md | 4 + notebooks/init.ipynb | 511 +++++++++++++++++- utils.py | 2 + 10 files changed, 559 insertions(+), 9 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/ml-diplom.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/other.xml create mode 100644 .idea/vcs.xml create mode 100644 utils.py diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..5c98b42 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,2 @@ +# Default ignored files +/workspace.xml \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..a184fa6 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/ml-diplom.iml b/.idea/ml-diplom.iml new file mode 100644 index 0000000..e1c70e3 --- /dev/null +++ b/.idea/ml-diplom.iml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..cc4ee99 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/other.xml b/.idea/other.xml new file mode 100644 index 0000000..68993fb --- /dev/null +++ b/.idea/other.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index e69de29..ee064f3 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,4 @@ +такс, ну я пытаюсь ставить pytorch, чорт как же он много весит, хорошо хоть данные на работе скачал + + +torch.cuda.is_available() -- false АЛЛО ЧИНИТЬ!!! \ No newline at end of file diff --git a/notebooks/init.ipynb b/notebooks/init.ipynb index a9b2291..7bb39f1 100644 --- a/notebooks/init.ipynb +++ b/notebooks/init.ipynb @@ -2,13 +2,506 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 3, + "metadata": { + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PyTorch Version: 1.2.0\n", + "Torchvision Version: 0.4.0a0\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "import numpy as np\n", + "import torchvision\n", + "from torchvision import datasets, models, transforms\n", + "import matplotlib.pyplot as plt\n", + "import time\n", + "import os\n", + "import copy\n", + "print(\"PyTorch Version: \",torch.__version__)\n", + "print(\"Torchvision Version: \",torchvision.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "data_dir = \"/home/nikita/PycharmProjects/hymenoptera_data\"\n", + "num_classes = 2\n", + "batch_size = 8\n", + "num_epochs = 15\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "def train_model(model, dataloaders, criterion, optimizer, num_epochs=25):\n", + " since = time.time()\n", + "\n", + " val_acc_history = []\n", + "\n", + " best_model_wts = copy.deepcopy(model.state_dict())\n", + " best_acc = 0.0\n", + "\n", + " for epoch in range(num_epochs):\n", + " print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n", + " print('-' * 10)\n", + "\n", + " # Each epoch has a training and validation phase\n", + " for phase in ['train', 'val']:\n", + " if phase == 'train':\n", + " model.train() # Set model to training mode\n", + " else:\n", + " model.eval() # Set model to evaluate mode\n", + "\n", + " running_loss = 0.0\n", + " running_corrects = 0\n", + "\n", + " # Iterate over data.\n", + " for inputs, labels in dataloaders[phase]:\n", + " inputs = inputs.to(device)\n", + " labels = labels.to(device)\n", + "\n", + " # zero the parameter gradients\n", + " optimizer.zero_grad()\n", + "\n", + " # forward\n", + " # track history if only in train\n", + " with torch.set_grad_enabled(phase == 'train'):\n", + " # Get model outputs and calculate loss\n", + " # Special case for inception because in training it has an auxiliary output. In train\n", + " # mode we calculate the loss by summing the final output and the auxiliary output\n", + " # but in testing we only consider the final output.\n", + " outputs = model(inputs)\n", + " loss = criterion(outputs, labels)\n", + "\n", + " _, preds = torch.max(outputs, 1)\n", + "\n", + " # backward + optimize only if in training phase\n", + " if phase == 'train':\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # statistics\n", + " running_loss += loss.item() * inputs.size(0)\n", + " running_corrects += torch.sum(preds == labels.data)\n", + "\n", + " epoch_loss = running_loss / len(dataloaders[phase].dataset)\n", + " epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n", + "\n", + " print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n", + "\n", + " # deep copy the model\n", + " if phase == 'val' and epoch_acc > best_acc:\n", + " best_acc = epoch_acc\n", + " best_model_wts = copy.deepcopy(model.state_dict())\n", + " if phase == 'val':\n", + " val_acc_history.append(epoch_acc)\n", + "\n", + " print()\n", + "\n", + " time_elapsed = time.time() - since\n", + " print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n", + " print('Best val Acc: {:4f}'.format(best_acc))\n", + "\n", + " # load best model weights\n", + " model.load_state_dict(best_model_wts)\n", + " return model, val_acc_history" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "#image_net = models.alexnet(pretrained=True)\n", + "#print(image_net)\n", + "\n", + "def set_parameter_requires_grad(model, feature_extracting):\n", + " if feature_extracting:\n", + " for param in model.parameters():\n", + " param.requires_grad = False" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Initializing Datasets and Dataloaders...\n" + ] + } + ], + "source": [ + "\n", + "\n", + "use_pretrained = False\n", + "feature_extract = True\n", + "model_ft = models.alexnet(pretrained=use_pretrained)\n", + "set_parameter_requires_grad(model_ft, feature_extract)\n", + "num_ftrs = model_ft.classifier[6].in_features\n", + "model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n", + "input_size = 224\n", + "\n", + "# Data augmentation and normalization for training\n", + "# Just normalization for validation\n", + "data_transforms = {\n", + " 'train': transforms.Compose([\n", + " transforms.RandomResizedCrop(input_size),\n", + " transforms.RandomHorizontalFlip(),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n", + " ]),\n", + " 'val': transforms.Compose([\n", + " transforms.Resize(input_size),\n", + " transforms.CenterCrop(input_size),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n", + " ]),\n", + "}\n", + "\n", + "print(\"Initializing Datasets and Dataloaders...\")\n", + "\n", + "# Create training and validation datasets\n", + "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'val']}\n", + "\n", + "# Create training and validation dataloaders\n", + "dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'val']}\n", + "\n", + "# Detect if we have a GPU available\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, "metadata": { - "collapsed": true, "pycharm": { + "is_executing": false, "name": "#%%\n" } }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Params to learn:\n", + "\t classifier.6.weight\n", + "\t classifier.6.bias\n" + ] + } + ], + "source": [ + "# Send the model to GPU\n", + "model_ft = model_ft.to(device)\n", + "\n", + "# Gather the parameters to be optimized/updated in this run. If we are\n", + "# finetuning we will be updating all parameters. However, if we are\n", + "# doing feature extract method, we will only update the parameters\n", + "# that we have just initialized, i.e. the parameters with requires_grad\n", + "# is True.\n", + "params_to_update = model_ft.parameters()\n", + "print(\"Params to learn:\")\n", + "if feature_extract:\n", + " params_to_update = []\n", + " for name,param in model_ft.named_parameters():\n", + " if param.requires_grad == True:\n", + " params_to_update.append(param)\n", + " print(\"\\t\",name)\n", + "else:\n", + " for name,param in model_ft.named_parameters():\n", + " if param.requires_grad == True:\n", + " print(\"\\t\",name)\n", + "\n", + "# Observe that all parameters are being optimized\n", + "optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0/14\n", + "----------\n", + "train Loss: 0.6926 Acc: 0.5451\n", + "val Loss: 0.6925 Acc: 0.6667\n", + "\n", + "Epoch 1/14\n", + "----------\n", + "train Loss: 0.6932 Acc: 0.4877\n", + "val Loss: 0.6928 Acc: 0.4771\n", + "\n", + "Epoch 2/14\n", + "----------\n", + "train Loss: 0.6931 Acc: 0.4795\n", + "val Loss: 0.6927 Acc: 0.5033\n", + "\n", + "Epoch 3/14\n", + "----------\n", + "train Loss: 0.6929 Acc: 0.5123\n", + "val Loss: 0.6930 Acc: 0.4575\n", + "\n", + "Epoch 4/14\n", + "----------\n", + "train Loss: 0.6928 Acc: 0.5082\n", + "val Loss: 0.6928 Acc: 0.4575\n", + "\n", + "Epoch 5/14\n", + "----------\n", + "train Loss: 0.6926 Acc: 0.5451\n", + "val Loss: 0.6932 Acc: 0.4575\n", + "\n", + "Epoch 6/14\n", + "----------\n", + "train Loss: 0.6928 Acc: 0.5246\n", + "val Loss: 0.6932 Acc: 0.4575\n", + "\n", + "Epoch 7/14\n", + "----------\n", + "train Loss: 0.6925 Acc: 0.5246\n", + "val Loss: 0.6925 Acc: 0.4967\n", + "\n", + "Epoch 8/14\n", + "----------\n", + "train Loss: 0.6928 Acc: 0.4836\n", + "val Loss: 0.6932 Acc: 0.4575\n", + "\n", + "Epoch 9/14\n", + "----------\n", + "train Loss: 0.6928 Acc: 0.5205\n", + "val Loss: 0.6929 Acc: 0.4575\n", + "\n", + "Epoch 10/14\n", + "----------\n", + "train Loss: 0.6922 Acc: 0.5533\n", + "val Loss: 0.6931 Acc: 0.4575\n", + "\n", + "Epoch 11/14\n", + "----------\n", + "train Loss: 0.6926 Acc: 0.5410\n", + "val Loss: 0.6933 Acc: 0.4575\n", + "\n", + "Epoch 12/14\n", + "----------\n", + "train Loss: 0.6926 Acc: 0.5205\n", + "val Loss: 0.6939 Acc: 0.4575\n", + "\n", + "Epoch 13/14\n", + "----------\n", + "train Loss: 0.6928 Acc: 0.5082\n", + "val Loss: 0.6935 Acc: 0.4575\n", + "\n", + "Epoch 14/14\n", + "----------\n", + "train Loss: 0.6923 Acc: 0.5246\n", + "val Loss: 0.6930 Acc: 0.4575\n", + "\n", + "Training complete in 1m 28s\n", + "Best val Acc: 0.666667\n" + ] + } + ], + "source": [ + "# Setup the loss fxn\n", + "criterion = nn.CrossEntropyLoss()\n", + "\n", + "# Train and evaluate\n", + "model_ft, hist = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft, num_epochs=num_epochs)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0/14\n", + "----------\n", + "train Loss: 0.6926 Acc: 0.4918\n", + "val Loss: 0.6923 Acc: 0.6340\n", + "\n", + "Epoch 1/14\n", + "----------\n", + "train Loss: 0.6926 Acc: 0.5533\n", + "val Loss: 0.6927 Acc: 0.5033\n", + "\n", + "Epoch 2/14\n", + "----------\n", + "train Loss: 0.6932 Acc: 0.5164\n", + "val Loss: 0.6930 Acc: 0.4575\n", + "\n", + "Epoch 3/14\n", + "----------\n", + "train Loss: 0.6928 Acc: 0.5287\n", + "val Loss: 0.6931 Acc: 0.4575\n", + "\n", + "Epoch 4/14\n", + "----------\n", + "train Loss: 0.6927 Acc: 0.5000\n", + "val Loss: 0.6921 Acc: 0.6928\n", + "\n", + "Epoch 5/14\n", + "----------\n", + "train Loss: 0.6924 Acc: 0.5410\n", + "val Loss: 0.6926 Acc: 0.4902\n", + "\n", + "Epoch 6/14\n", + "----------\n", + "train Loss: 0.6921 Acc: 0.5410\n", + "val Loss: 0.6928 Acc: 0.4575\n", + "\n", + "Epoch 7/14\n", + "----------\n", + "train Loss: 0.6922 Acc: 0.5902\n", + "val Loss: 0.6926 Acc: 0.4641\n", + "\n", + "Epoch 8/14\n", + "----------\n", + "train Loss: 0.6922 Acc: 0.5533\n", + "val Loss: 0.6924 Acc: 0.4967\n", + "\n", + "Epoch 9/14\n", + "----------\n", + "train Loss: 0.6928 Acc: 0.5082\n", + "val Loss: 0.6931 Acc: 0.4575\n", + "\n", + "Epoch 10/14\n", + "----------\n", + "train Loss: 0.6929 Acc: 0.5410\n", + "val Loss: 0.6931 Acc: 0.4575\n", + "\n", + "Epoch 11/14\n", + "----------\n", + "train Loss: 0.6925 Acc: 0.5943\n", + "val Loss: 0.6925 Acc: 0.4706\n", + "\n", + "Epoch 12/14\n", + "----------\n", + "train Loss: 0.6922 Acc: 0.5041\n", + "val Loss: 0.6915 Acc: 0.6993\n", + "\n", + "Epoch 13/14\n", + "----------\n", + "train Loss: 0.6924 Acc: 0.5287\n", + "val Loss: 0.6924 Acc: 0.4837\n", + "\n", + "Epoch 14/14\n", + "----------\n", + "train Loss: 0.6921 Acc: 0.5779\n", + "val Loss: 0.6923 Acc: 0.4706\n", + "\n", + "Training complete in 1m 31s\n", + "Best val Acc: 0.699346\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXgUVdb48e9JQghhX0X2XdYAIWyiiIoKOoCjg8io4zbi6LiNo47v+JNRRn1H3HXcGHdF3BeG1wVRFGTfkUUkrAlrAAMEwpLk/P64FWhCJ+kkvSTp83meftLVVX3rdKe6TtWtW/eKqmKMMSZ6xUQ6AGOMMZFlicAYY6KcJQJjjIlylgiMMSbKWSIwxpgoZ4nAGGOinCWCAIhIKxFREYnzpr8UkasDWbYU6/q7iLxSlnhNxVfW7SgI6x8gImtFJEtELg7xumK99bQI5rIVgYi8IyIPRDqOqEgEIvK1iIzz8/oIEdle0h+bqg5V1TeDENcgEUkvUPYjqvrHspZdzDpVRO4J1ToqIxG5xvve7i7werqIDIpQWKE0Dvi3qtZQ1c98Z3g74vxHnohk+0xfUdIVqWqut57NwVy2pETkIRE5WuDz7Qr2esqjqEgEwBvAVSIiBV6/CpioqjnhDylirgb2eH/DKlJHt0G0B/ibiNSKdCAlUcrvvSWw0t8Mb0dcQ1VrAJuBYT6vTQzS+iNlou/nU9UGkQ4oHKIlEXwG1APOzH9BROoCvwHe8qYvEpElIrJPRNKKOl0Tke9F5I/e81gReVxEdonIeuCiAsteKyKrRWS/iKwXkRu916sDXwJNfI4+mojIAyLyjs/7h4vIShHJ9NbbyWfeRhG5S0SWi8heEXlfRBKKiDsR+B3wZ6C9iKQUmH+GiMz21pUmItd4r1cTkSdEZJO3nh+91046o/FiGuw9f0BEPvJOf/cB14hIHxGZ461jm4j8W0Tifd7fRUS+EZE9IrLDqyprLCIHRaS+z3K9RCRDRKoUWH8T7wi1ns9rPb3/TxURaSciP3ifY5eIvF/Y9+XHamAO8JdCvt83ROQhn+kTvh/vu7nb+38dEJFXReQUcVWN+0Vkmrdd+rpORLZ639VffcqKEZF7RWSdiOwWkQ/yP7Mcr1a6XkQ2A98VEu8NIpLqfdeTRaSJ9/o6oA3wX2+7rFqC7yj/yPp9EZkkIvuBK0Wkv4jM9fm/P5v/vxOROC/eVt70O978/O9ljoi0Lumy3vyhIvKL9/9+TkRm5W/XJfxM+eu9VUQ2eNvOv0QkxpsfIyJjvd/ITm9bqOXz/oHe598r7rd1lU/x9Qr5rDHeZ9vpvW+5iHQuaewBUdWoeAD/AV7xmb4RWOozPQjohkuOScAO4GJvXitAgThv+nvgj97zPwE/A81xyWZ6gWUvAtoCApwFHASSfdaZXiDOB4B3vOcdgAPAeUAV4B4gFYj35m8E5gNNvHWvBv5UxHdwFbANiAX+CzzrM68FsB8Y7a2rPtDDm/e895mbeu89HahaSPwbgcE+n+UocLH3vVYDegH9gDjve10N3OEtX9OL769Agjfd15v3BXCTz3qeAp4r5HN+B9zgM/0Y8JL3fBJwnxdPAnBGgNvPNcCPQA8gE6jnvZ4ODPKevwE8VGCbSi/w3cwFTvG+y53AYqCn931+B/yjwDY3CaiO2zYzfL7bO7yymnnvfRmYVOC9b3nvrebn85wD7AKSvfc/B8zw938s5ns5aTngIeAIMMzn/94b6Ov939sAvwC3eMvHefG28qbf8WJLwW2L73P8N1GSZRvhtukR3rw7cdvjNYV8loeANwqZl7/eaUBd7ztOzS8LGON9pta47fZz4HVvXmsvjsu8chpw/LdVVPwX4X7ftb3vsTPQOCT7x1AUWh4fwBnA3vwfBTAL+EsRyz8NPFXgh+UvEXyHz84XON93WT/lfgbc7j0fRNGJ4H7gA595McAWju94NgJX+swfj7fDK2Td04CnveejcTuWKt70/wCf+nlPDJANdPczz1/8GzkxEcwoLB5vmTvy1+vFtKSQ5UYBs7znscB2oE8hy/4R+M57LkAaMNCbfguYADQr4fZzDfCj9/wD4FHveUkTwRU+0x8DL/pM3wp8VmCb61jg//uq93w1cK7PvFNxO7k4n/e2KeLzvAqM95mu4b2/VcH/YzHfy0nL4Xao3xXzvruAD73n/nbuL/ksOxxYUYplrwNm+swT3IHGNYXElJ/AMn0e3xRY72Cf5W8Dvvae/wCM8ZnXBTiM+/3cn/9Z/ayzqPjPxx1k9gViSrK9lvQRLVVDqOqPuB3fCBFpgztCeTd/voj0FZHpXnXDXtyRfiD1g01wO5p8m3xneqemc73T70zgwgDLzS/7WHmqmuetq6nPMtt9nh/E/aBPIiLNgbOB/Drcz3FHxPlVWc2BdX7e2sBbzt+8QPh+N4hIBxGZIu4i/T7gEY5/H4XFkB9vZ+9/dx6wV1XnF7LsR0B/r6pjIO4HPNObdw9uhzBfXJXbdaX4TGOBm0SkcSneu8Pnebaf6YL/v4LbVhPveUvgU6+qJROXGHJxZxv+3ltQwW0rC9jNidtWWRT8v3cUkf/z+b+Po+jfQUDbdTHLnvDbVLd3PaEq0493VbWOz+O8AvML+3+c8H16z+OBhhS9XRcav6pOBV4CXgR2iMhLIlKzmPhLJWoSgect4A+4KpKpqur7I3wXmAw0V9XauH9AwYvL/mzD/aPzHWvW5tWtfgw8DpyiqnVwVRz55WoxZW/F/eDzyxNvXVsCiKugq3D/7/+KyHZgPW4H/wdvfhquCqugXcChQuYdABJ94ovFbfi+Cn7GF3FHOe1VtRbwd45/H4XFgKoewh2JX+F9lrf9LectmwlMxZ2K/x5XZaLevO2qeoOqNsFVD74gIu0KK6uQ8n8GPvFi93XC9wGUJlEUVHDb2uo9TwOGFthpJaiq77ZR1PZVcNuqjqsOLM225U/Bdb8MrADaef/3sQT2+yqLbbiqM+DY76esia6w/8cJ36c37wju4LPQ7bo4qvq0qiYDXXFVQ3eWppziRGMiGAzcABRs/lkT2KOqh0SkD24HEogPgNtEpJl3oe9en3nxuPrXDCBHRIbiTvfy7QDqi0jtIsq+SETO9S6s/RV3ujk7wNh8/QF4EFfHnf+41Cu/Pu5MYbCIXOZdGKsvIj28s5DXgCfFXYiN9S78VcXViSaIu9BeBfh/3uctSk1gH5AlIh2Bm3zmTQEai8gdIlJVRGqKSF+f+W/hqmiG406pi/Ku95kv5cQzv5Eikr9z+BW3w8otpix/HgSuBer4vLYUuFBE6nlnC3eUotyC7heRRBHp4q0v/+L2S8DDItISQEQaisiIEpT7LnCtiPTw/pePAPNUdWMQYvanJq5q9oC4Bg83hmg9vqYAySIyTFzLpds5+UClpO4RkTri7mO4jeP/j0nAneIu1NcEHsYdgOThttUhInKp99tqICLdi1uRuIYVfbzYD+ASS2m21WJFVSLwNvLZuAtokwvMvhkYJ66Vw1jcTjgQ/wG+BpbhLvx94rO+/biN5QPcTuf3vuv1jiwnAeu9U/wmPuWiqmuAK3EX8nbhLr4NU9UjAcYGgIj0w9UbP+8dEec/JuMueI1W1y77Qlyy2YPbqeVvrHcBPwELvHmP4uos9+K+t1dwR5IHKP7U+y7ve9iP++6Otdrxvq/zvM+5HViLq87Knz8LyAMWB7DDmgy0B3ao6jKf13sD80Qky1vmdlXd4H1PKyXAdvDee97GbUv53sZtBxtxZyQlaZFUmB9w/6Nvgce96gKAZ7z4p3rb7FxcXXJAVPVbXN31x7gj57bA5UGItzB/xTVZ3o87OwjGd1Mk74x/FPAkrtqrLbAEdzBVmCvkxPsIssSntRqukcVSr5xPcdeF4Pi2PBN3tr0fl3jyt5VhwN9wv5/FuIv/xamDu5aTidumtuEaSQSdeGfMxlQIIvIdrh7X7r42JeJVXW4FfqeqM4tbvsB743AX01uH8KwpYqLqjMBUbCLSG9fcMeRHk6ZyEJEhIlLbq/66H8jBNck0PkKWCETkNe9GiBWFzBfvZolU70aJ5FDFYio+EXkT1/z1Dq8KyZhAnIGrqtkFDMHdG1RU1VBUClnVkIgMBLKAt1S1q5/5F+LaTV+Iq9t8RlUDruM0xhgTHCE7I1DVGbgLI4UZgUsSqqpzgToicmqo4jHGGONfJDuDasqJN2eke69tK7igiIzB3cJN9erVe3Xs2DEsARpjTGWxaNGiXarqt/lsJBOBv5tJ/NZTqeoEXLcApKSk6MKFC0MZlzHGVDoisqmweZFsNZTOiXfpNeP4XXrGGGPCJJKJYDLwB6/1UD9c3zEnVQsZY4wJrZBVDYnIJFzviw3E9cn+D1w3q6jqS7g+dy7E3TV5EHf7vDHGmDALWSJQ1dHFzFfcACnGmChz9OhR0tPTOXToUKRDqXQSEhJo1qwZVapUKX5hT0UaQs4YU0mkp6dTs2ZNWrVqhZw0gqwpLVVl9+7dpKen07p16+Lf4LEuJowxYXfo0CHq169vSSDIRIT69euX+EzLEoExJiIsCYRGab5XSwTGGBPlLBEYY6JSbGwsPXr0oGvXrowcOZKDBw+W6P1PP/10id8DMHbsWKZNm1bi9/kzaNAggnGDrSUCY0xUqlatGkuXLmXFihXEx8fz0ksvnTBfVcnLyyv0/UUlgtzcwgcSGzduHIMHDy5d0CFiicAYE/XOPPNMUlNT2bhxI506deLmm28mOTmZtLQ0pk6dSv/+/UlOTmbkyJFkZWXx7LPPsnXrVs4++2zOPtsNolejRg3Gjh1L3759mTNnDuPGjaN379507dqVMWPGkN/T8zXXXMNHH30EQKtWrfjHP/5BcnIy3bp14+effwbgwIEDXHfddfTu3ZuePXvy+eefA5Cdnc3ll19OUlISo0aNIjs7Oyif35qPGmMi6sH/rmTV1n1BLbNzk1r8Y1iXgJbNycnhyy+/ZMiQIQCsWbOG119/nRdeeIFdu3bx0EMPMW3aNKpXr86jjz7Kk08+ydixY3nyySeZPn06DRo0ANzOu2vXrowbN87F0LkzY8eOBeCqq65iypQpDBs27KT1N2jQgMWLF/PCCy/w+OOP88orr/Dwww9zzjnn8Nprr5GZmUmfPn0YPHgwL7/8MomJiSxfvpzly5eTnBycYVwsERhjolJ2djY9evQA3BnB9ddfz9atW2nZsiX9+vUDYO7cuaxatYoBAwYAcOTIEfr37++3vNjYWC699NJj09OnT2f8+PEcPHiQPXv20KVLF7+J4JJLLgGgV69efPKJG/J86tSpTJ48mccffxxwzW03b97MjBkzuO222wBISkoiKSkpGF+FJQJjTGQFeuQebPnXCAqqXr36seeqynnnncekSZOKLS8hIYHY2FjA7bhvvvlmFi5cSPPmzXnggQcKbdtftWpVwCWSnJycY+v9+OOPOe20005aPhTNbu0agTHGFKJfv37MmjWL1NRUAA4ePMgvv/wCQM2aNdm/3/+oqfk7/QYNGpCVlXXsmkCgLrjgAp577rlj1xWWLFkCwMCBA5k4cSIAK1asYPny5SX/UH5YIjDGmEI0bNiQN954g9GjR5OUlES/fv2OXdAdM2YMQ4cOPXax2FedOnW44YYb6NatGxdffDG9e/cu0Xrvv/9+jh49SlJSEl27duX+++8H4KabbiIrK4ukpCTGjx9Pnz59yv4hCeGYxaFiA9MYU/GtXr2aTp06RTqMSsvf9ysii1Q1xd/ydkZgjDFRzhKBMcZEOUsExhgT5SwRGGNMlLNEYIwxUc4SgTHGRDlLBMaYqPTwww/TpUsXkpKS6NGjB/PmzStTeZmZmbzwwgvFLhesrqODyRKBMSbqzJkzhylTprB48WKWL1/OtGnTaN68ebHvy+8Cwp9AE0F5ZInAGBN1tm3bRoMGDY7189OgQQOaNGnCggULOP300+nevTt9+vRh//79vPHGG4wcOZJhw4Zx/vnnk5WVxbnnnnus6+j8LqLvvfde1q1bR48ePbj77rsBGD9+PN26daN79+7ce++9x9b/4Ycf0qdPHzp06MDMmTPD/wUUYJ3OGWMi68t7YftPwS2zcTcY+q9CZ59//vmMGzeODh06MHjwYEaNGkX//v0ZNWoU77//Pr1792bfvn1Uq1YNcGcQy5cvp169euTk5PDpp59Sq1Ytdu3aRb9+/Rg+fDj/+te/WLFixbGO7L788ks+++wz5s2bR2JiInv27Dm2/pycHObPn88XX3zBgw8+GLQRy0rLEoExJurUqFGDRYsWMXPmTKZPn86oUaO47777OPXUU4/1C1SrVq1jy5933nnUq1cPcD2D/v3vf2fGjBnExMSwZcsWduzYcdI6pk2bxrXXXktiYiLAsffDiV1Pb9y4MVQfM2CWCIwxkVXEkXsoxcbGMmjQIAYNGkS3bt14/vnnC+3i2bdr6okTJ5KRkcGiRYuoUqUKrVq18tvFtKoWWp6/rqcjya4RGGOizpo1a1i7du2x6aVLl9KpUye2bt3KggULANi/f7/fnfTevXtp1KgRVapUYfr06WzatAk4uVvq888/n9dee+3YuMa+VUPljZ0RGGOiTlZWFrfeeiuZmZnExcXRrl07JkyYwLXXXsutt95KdnY21apV81t3f8UVVzBs2DBSUlLo0aMHHTt2BKB+/foMGDCArl27MnToUB577DGWLl1KSkoK8fHxXHjhhTzyyCPh/qgBsW6ojTFhZ91Qh5Z1Q22MMaZELBEYY0yUs0RgjImIilYtXVGU5nu1RGCMCbuEhAR2795tySDIVJXdu3eTkJBQovdZqyFjTNg1a9aM9PR0MjIyIh1KpZOQkECzZs1K9B5LBMaYsKtSpQqtW7eOdBjGY1VDxhgT5UKaCERkiIisEZFUEbnXz/wWIjJdRJaIyHIRuTCU8RhjjDlZyBKBiMQCzwNDgc7AaBHpXGCx/wd8oKo9gcuBitmZtzHGVGChPCPoA6Sq6npVPQK8B4wosIwC+V381Qa2hjAeY4wxfoQyETQF0nym073XfD0AXCki6cAXwK3+ChKRMSKyUEQWWisDY4wJrlAmAn/9rxZsNDwaeENVmwEXAm+LyEkxqeoEVU1R1ZSGDRuGIFRjjIleoUwE6YDvIKDNOLnq53rgAwBVnQMkAA1CGJMxxpgCQpkIFgDtRaS1iMTjLgZPLrDMZuBcABHphEsEVvdjjDFhFLJEoKo5wC3A18BqXOuglSIyTkSGe4v9FbhBRJYBk4Br1O45N8aYsArpncWq+gXuIrDva2N9nq8CBoQyBmOMMUWzO4uNMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6JcsYnAG3LSGGNMJRXIGUGqiDzmZ7xhY4wxlUAgiSAJ+AV4RUTmesNG1iruTcYYYyqGYhOBqu5X1f+o6unAPcA/gG0i8qaItAt5hMYYY0IqoGsEIjJcRD4FngGeANoA/6XAWAPGGFNh/PAYvHQG5B6NdCQRF0jV0FpgBPCYqvZU1SdVdYeqfgR8FdrwTIWQsQbmvAA2uJypKA5nweznYPtPsPLTSEcTcYGMUJakqln+ZqjqbUGOx1REX98Hqd/AKV2gzVmRjsaY4i2bBIf3QmJ9mPUMdBsJIpGOKmICOSN4XkTq5E+ISF0ReS2EMZmKZPc6lwQAfhgf2ViMCUReHsx9EZr2gvP+CTtWQOq0SEcVUQG1GlLVzPwJVf0V6Bm6kEyFMv8/EFMFzrgTNv0IG3+MdETGFC31G9izDvrd7M4EajV1ZwVRLJBEECMidfMnRKQeIR703lQQh7Ng6UToPALOugdqnAI/PBrpqIwp2twXoeapbruNi3cJYeNMSF8U6cgiJpBE8AQwW0T+KSL/BGYDFbIOIPtIbqRDqFyWvweH90HfG6FKNRhwO2yYAZvmRDoyY/zbuRrWT4c+N0BsFfdar6shoTbMeiqysUVQIPcRvAX8DtgB7AQuUdW3Qx1YsL05eyODHp9O1uGcSIdSOai6aqFTe0Cz3u61XtdC9YZ2VmDKr7kvQlyC21bzVa0Jvf8Iq6fArtTIxRZBAXU6p6orgQ+Az4EsEWkR0qhCIKlZbXbsO8xrP26IdCiVw4YfIONndzaQ39oiPhFOv80dcaXNj2x8xhR0YDcsfx+6Xw6J9U6c1/dPEBsPs5+NTGwRFsgNZcNFZC2wAfgB2Ah8GeK4gq5ni7qc3/kU/jNjPb8eOBLpcCq+eRNc07sul5z4esp17nVrQWTKm0WvQ84h6HvTyfNqNIIev3fNSvdvD39sERbIGcE/gX7AL6raGjgXmBXSqELkrgtOI+tIDi/+sC7SoVRsv26CX76E5KuhSsKJ86rWgP63uJYZUXzxzZQzOUdgwSvQ5mxo1NH/MqffCnk5MO+l8MZWDgSSCI6q6m5c66EYVZ0O9AhxXCHR4ZSaXNKzGW/O3sj2vYciHU7FtfBVQKD39f7n97kBqtWFGXZWYMqJVZ/D/m2uhVBh6reFTsNhwWtwaF/4YisHAkkEmSJSA5gBTBSRZ4AKe8X1jsHtyVPlmW/XRjqUiuloNix+CzpeBLWb+V+mak3o/2f45SvYujS88RlTkCrMfR7qt4N2g4tedsDt7o7jRa+HJ7ZyIpBEMAI4CPwF17fQOmBYKIMKmbxcmtdL5Iq+LflgYRobdh2IdEQVz08fQvav7iJxUfqMcU3y7FqBibS0+bB1ibsgHFPMLq9pMrQe6FoX5RwOT3zlQJHfijc62eeqmqeqOar6pqo+61UVVSzLP4CXz4LD+/nz2e2oGhfDk9/8EumoKhZVmD8BGnWGlgOKXjahNvT7M6z5P9i2PDzxGePPvBfd9th9dGDLD7jDVSMt/yC0cZUjRSYCVc0FDopI7TDFEzq1msLOVTD5VhrWiOe6Aa3577KtrNy6N9KRVRyb57reGvuMCayDrr43QtVaMOOx0MdmjD+ZabBqsmvYULVGYO9pew407ua6ncjLC2185UQgVUOHgJ9E5FUReTb/EerAgq7VADj3ftfl7PwJ3DCwDbWrVeHxr9dEOrKKY/7L7sgq6bLAlq9Wx52Or54MO1aGNjZj/FnwH/e3z5jA3yPizgp2r3Wt46JAIIng/4D7cReLF/k8Kp7Tb4cOQ+Hr+6i9exk3DWrL9DUZLNi4J9KRlX/7trojq55XQXz1wN/X7yaIr2lnBSb8jhyARW9Ap2FQp3nJ3tv5YqjTAn58OirG2Qiki4k3/T3CEVzQxcTAb1+EWqfCB1dzdfdaNKpZlfFf/YxGwT+7TBa+DprnbsUvicR60HcMrPwMdv4cmtiM8WfZJDi01x2MlFRsHPS/FdLnw+bK33dWIHcWbxCR9QUfgRQuIkNEZI2IpIrIvYUsc5mIrBKRlSLybkk/QIlVqwuXvQUHdlJtyp+47Zy2LNj4K9+vyQj5qiusnMOuOV2HC6Be65K/v9+foUqinRWY8MnLg7kvQZOe0Lxv6croeeXxgWsquUCqhlKA3t7jTOBZ4J3i3uS1OHoeGAp0BkaLSOcCy7QH/gcYoKpdgDtKFH1pNekJQx+F1GmMPvwhLesnMv7rNeTl2VmBXys/gwMZJatn9VW9vrvJbMXHkGEttUwYrPvW1fH3u7n0I4/FJ0KfG939MDtWBTe+ciaQqqHdPo8tqvo0cE4AZfcBUlV1vaoeAd7D3ZPg6wbgeW+wG1R1ZwnjL71e10K3y4j9/hH+t/suVm/bx5SftoVt9RXK/Jehfnt3e35p9b/FdVU984ngxWVMYea+ADUau7r+suhzgzubreSd0QVSNZTs80gRkT8BNQMouymQ5jOd7r3mqwPQQURmichcERlSSAxjRGShiCzMyAhSFY4I/OYpaHga/Zfey4BGR3hy6hqO5kZHc7GApS+CLYvc2UBxN+MUpUZD1yHdTx+44S2NCZWdP8O676DPH93AM2WRWA+S/+BupNybHpz4yqFAB6bJf/wvkAwE0n7Q3/lYwbqXOKA9MAgYDbziOz7ysTepTlDVFFVNadiwYQCrDlDVGnDZW8jRbJ6Pf4703fv4cGHl/WeXyvwJrtVPjwBvxinKgNtdV792VmBCad5LJ485UBb9/+xaDs15ITjllUOBVA2d7fM4T1XHqGogje/TAd82W82ArX6W+VxVj6rqBmANLjGET8PTYPiz1Nm1iCfqfc4z3/7CoaM2khkAWRmw8hOXBKoGchJYjBqN3FnBsvdgj40LYULg4B63fSVdBtUbBKfMOi2g66WuKerBytnUPJCqoUd8j9JFpK6IPBRA2QuA9iLSWkTigcuByQWW+Qw42yu3Aa6qKKAWSUHV7XfQ+wZGHPyYHlk/8tacjWEPoVxa9AbkHin9RWJ/Tr8NYuLsrMCExqI3ICfb/5gDZTHgdjh6wOt5t/IJpGpoqKpm5k94F3YvLO5NqpoD3AJ8DawGPlDVlSIyTkSGe4t9DewWkVXAdODuiPVjdMHD0CSZp6q+zOTps9h36GhEwig3co+6jb7tOdAgiCdptU51Y8Qum+TGNTAmWHKPuuFTW58Fp3QufvmSaNwV2p3nmqQezQ5u2eVAIIkgVkSq5k+ISDWgahHLH6OqX6hqB1Vtq6oPe6+NVdXJ3nNV1TtVtbOqdlPV90rzIYIiriqMfIP4+Co8mvs4b0yv3M3FivXzFNfxVp9iehktjQF3gMTAj9E7WLgJgVWfw/6tRY85UBYDboeDu2DpxNCUH0GBJIJ3gG9F5HoRuQ74BqiYdxYXp25L4i79D11iNtF47oPsyoqebmhPMm8C1G0F7c8Lftm1m7quKpa84zoFMyYY5r4I9dpA+/NDU36rM6BpL5j9HORVruuIgVwsHg88BHQCugD/9F6rnDpcwK+9buUy+ZaZH1butsOF2v4TbJ7tupOIiQ3NOs74i/s76+nQlG+iS9oC2LLQXRsoSzPnouR3RvfrRnf2UYkEcrG4NfC9qt6lqn8FZohIq1AHFkl1L3yAddWTGbJxPNvXVsz+9cpk3svuJpqeV4ZuHXWaQ88r3Ghn+wo2JjOmhOa+AFVruwHoQ6njRVCvrTuAqUT9kwWSOj8EfO+yyoF2LtIAAB7xSURBVPVeq7xi46hxxRvsJ5HYD/8QXeOXHtzjbp5Jusz1yxRKZ9zpOrL70c4KTBnsTXdH6MlXBT7mQGnFxMKA22DbMtjwQ2jXFUaBJII4r4sIALznZbxdr/w7pUlLvur4CHUPb2X/hzdVquxfpCVvQ86h4DYZLUzdltD9ctfkb//20K/PVE4LXgE0PNssQNLlUOOUSnUAE0giyPBp7omIjAB2hS6k8uM3w0fyDKOpuW6Ku8O2ssvLdT+qlmfAKV3Cs84z/wp5OTArSq/HmLI5ctB1kd7xN+7AIhyqJLgBl9ZPh61Lw7POEAskEfwJ+LuIbBaRNOBvQAjaFJY/9arHE3fG7XyT24u8r+9zF6Qqs1++gszNbvyAcKnXBpJGwcLXICt8fQ6aSmL5e3Aos3RjDpRFynWu65VK0hldIK2G1qlqP1xX0p1V9XRgf8gjKyeuH9iWh6vcyi6pDx9eAwcic79bWMx7GWo1g9MuCu96B94FuYcrzY/KhElenmsyemp3aNE/vOuuVgdSrnVD31aC7lJK0s4qFhgpItOAxSGKp9ypUTWOq87pwXUHbyEvayd8ckPlHNA6Y427+NX7Ojc6UzjVbwvdRsKCV+FAVNQ6mmBY/x3s+qVsYw6URb+bQGJhzr/Dv+4gKzIRiEg1ERklIp8DK4AncfcUlHAA0Irtir4t2FOrMy8ljnEDXsx8PNIhBd/8CRBbFZKvjsz6z7zL3bo/+7nIrN9UPHNfdBdtu/w2Muuv1QS6j3I3RlbwA5hCE4GITAR+Ac4H/g20An5V1e9VtRIeEhcuoUosdwzuwPhd/dnaYjhMfwTWTY90WMFzaC8sneR6WAxWj40l1bADdL3E9RVTmavfTHBkrIHUae6mx7iAerwJjdNvd0O5zns5cjEEQVFnBF2BX3Edxv2sqrmcPJ5A1LgkuSltG9bgxl+vRBt2hI//WHluhFr6rutZMZwXif0ZeDccPehuDjKmKPNecmewwRpzoLQadnA3mc2fAIezIhtLGRSaCFS1O24AmlrANBGZCdQUkcbhCq48iYuN4a/nn8ZPGTl802W8q8b48FrX42FFlpfnjsKb9XFjOUdSo07QeYQ7uqqk/b6bIDi4x53BJo10I99F2oDbXculJW9HOpJSK/Iagar+7PUWehrwF+AtYL6IzA5LdOXM0K6N6da0NuPm5ZDzm2cgbS5MeyDSYZXNuu9gzzroW05aBJ91DxzZ7474jPFn8ZuhGXOgtJr3gRanw5znK+yBYcCthlR1odfXUEvgf0IXUvklItx9wWmk/5rNxAO9ofcNrsXAsveDeufx7qzDTF25nU27DwStzELNf9ldcOs0vPhlw+GULtBpmOv3PTuz+OVNdMkfc6DVmW6MgPLijDtgbxqs+DjSkZRKidsJqqoClaeTjRI6s30D+rWpx3PfpTLyzgdJ3LoEPh3jdqin3+Z2YiXssTPrcA7zN+xmVupuZqXu4uft7jaNGIEhXRszZmBbejQ/aSjnstu9DtZ+A2f9reyDfAfTwHtg9X9dFdGgv0U6GlOerP4v7NsCF5azlnvtzoOGnWDWM+4GyUg0Zy2DMDcYr/hEhHuGdOSSF2bz+rxt/PmaKe5i65x/w4dXQ93WbrDrHldAfKLfMg7n5LJkcyazU3cxa91ulqVlkpOnxMfFkNKyLndfcBrJLeoyY20G78zdxBc/badP63rcOLANZ5/WiJiYIG1kC151SSslwhfcCjo1CU67EOY+79pqJ9SKdESFUlXmb9jDewvSmLt+N2e2b8Co3i1IblEHKcvOIDMNVk92nall7XTDqfa80o0REc3mvuh+Yx0uiHQkJ4qJcdcKPvuTO7jqEKIxEUJEtIJ1ppaSkqILFy6MdBj88c2FzNuwm5n3nE2dxHjXT8/PU1yfOVsWQmJ9V3XU5wZyq9Vn1dZ9zFq3i1mpu1iwcQ+HjuYRI9CtWR0GtK3PgHYN6NWyLglVTjybyDqcw3vzN/PajxvYuvcQ7RvV4IaBbRjRowlV48owVsDhLHiysxt45nflcBzWrUtgwiDXKmTQvVCzfLVRyNh/mI8Xp/PBgjTW7zpAzapx9G1Tj9nrdnPwSC4dTqnBqN4tuKRnU+pWD/Bsa896WOXt/Ld692w27gbVG8L67131Y5tBkPwH11Ilks0mIyF9IbxyLgx5FPr9KdLRnCz3KDzTw/V5dO0XkY7mJCKySFVT/M4rLhF4w1ReiruP4NgZhKqOC2KMASsviWDN9v0MeWYGNw5sy71DOx6foYpums3B75+i+sZvOCJV+VTP4oXDQ9ikjWnfqAYD2jXg9Lb16dumPrWrVQlofUdz85iyfCsv/7Cen7fvp1HNqlw7oDW/79si4DJOsOBV+L874fpv3MWu8ujTP7mxjRFo0c+1KOo0DGo3i0g4uXnKzLUZvDc/jWmrd5CTp/RuVZdRvVtwYbfGJMbHkXU4hynLtvLegjSWpmUSHxvDBV0bc3nv5vRvU//ks7mMX2D1527nv/0n91qTZOg83F23qd/WvbY3HZZMdC1T9qa5A43uo11SaHhaeL+ISPnoelg7Fe5cBVVrRjoa/+Y8D1//Ha772m2z5UhZE8FXwF5gEW4sAgBU9YlgBhmo8pIIAP7y/lK+XLGNGXefTa4qs1J3Mzt1F7PX7Wb7vkO0lS3ckfgVQ/NmEKu5HG5/EQln/QWa9Sr1OlWVmWt3MWHGen5M3UWNqnFc3rs5153RmiZ1qgVaCLzQ310XGPND+a7P3Pmz20mu+hx2rnSvNU1xSaHz8LBUlWzJzOaDBWl8uDCNrXsPUa96PJcmN2VU7+a0a1T4Dmn1tn28vyCNT5dsYW/2UVrUS2RUSjNGtcqiwaav3GfKWO0Wbt7X7fg7DSu6F828XNfr5eK34OcvIO8oNO/nEkKXiyG+epA/fTmxdws8k+TG0B7ySKSjKdzhLHiqi2tOWq2e2z7rtnR/67Q8Pl27OcSW4gCuDMqaCFaoarm5PF+eEsHm3Qc554nvSYyPZd+hHMD1WNq/bX0GtHVH/S3rJyJZO1xzyAWvweG90HKAu7Dc/vwyDau3Yste/jNzPVOWb0OAYd2bMGZgGzqdWkyd+vof4K3hMOIFN0pYRbEr9fjR87Zl7rVTu3tnCiOgQbugrepobh7frt7BpPlpzFibAcAZ7Rpwee8WnNf5FOLjAv+/HTqSw9zZ0/l14Uck7fuBtjHbyEPY2zCFWsm/I7bLcNddQUllZbgzpsVvwe61ULWWu5aQ/IfI3xMSbNMedKOC3bak/F8n2bna9eT76yY3rGXmJnfNJ8+naanEuA4e67b0SRStjieK6g2DfoBW1kQwAXhOVX8KalSlVJ4SAcCEGeuYt34P/dvW5/S2DejYuGbhF3MP73c/2jkvwL50aHAanH6rGw2sDPW96b8e5NUfN/D+gjQOHsllYIeG3DiwDae3re//guV7V8DmOfCXVa5v9YpozwbXgmTV5+6aDECjLt6Zwgho1LHo9xdifUYW7y9I4+PF6ezKOkLjWglcltKMkSnNaV7P/8V/v1Rhy6LjZzOZm0BiyW7anxlVBvDk5vasOZDIKbWqMrJXcy5LaU6L+iUov+C6Ns917etXfuba2DdOcgmh20jXU2ZFdeSg25m+caE7gLp8YqQjKp28XNcTQaaXHPKTRH6iyNpx4vJVEk88g8h/3qQn1Dq1VCGUNRGsAtoBG4DDgOBakSaVKpoyKm+JoFRyj7rua2c9AztWQI3G7uJXr2vL9KPNPHiEifM28/qsjezKOkyXJrUYM7ANF3U7lbhY7wg2czM8090Nwj34H0H6QBG2N/14Utg8F1Bo0OF4Ujila5FHV4eO5vLlim1Mmp/G/A17iI0Rzu3YiMv7NOesDo2IDbSVVl4epM3zWvtMdsk+Js5d4O08wnXvXb0+4M44vvt5J+8vSOP7NTvJU3fGMap3c87vckrpGwJkZ7qhRhe/6a45xFVzVUbJf3BdNZe3asD8HWT+DtH3KPrXjT47SIFrv4SWYe5uOlyOHHS/zYKJIn/6iNd9xUVPuP6VSqGsicBvhaWqbipVNGVUKRJBPlV3Z+/sZ12rkPga0Osa12SyDBdEDx3N5bMlW5gwcz3rMw7QtE41rj+jNaN6N6f6jH+69d3xU8QuuobU/u3Hk8KmWW5M5Lqtj19TaJJ8bGe4aus+3luwmU+XbGH/oRxa1k9kVO/m/K5XMxolxrnO+A5luke29/fQ3uPPszOPL5Od6RLSwV0QGw9tz/V2/kOKHft5a2Y2Hy1K5/0FaWzJzKZuYhUuSW7G5b2b0/6UMlwU3brEnYEu/9DdrV2/vUsI3UeHr2sGVcj+9cSdWkmqTPKrSxp3dV2QRCNV163GrxuhdtNSt6ArUyLwCugOnOlNzlTVZaWKJAgqVSLwtW2Z64J5xSduR9X10jLX8+ap8suO/cxet5u0PQdJqBLLrXGfsrNuCt91j8i1/rBKOLKHFhnTabVjGk1+nU+M5rI/4VTWNzibFXtiyN63m7oxB2lfK5eW1Y9Siywke6/buR8pZuyl2HhIqOPO4BJqu+fVG0K7c921n1Lc+5CXp/yYuov3F6QxddV2juYqvVrWddckYkt/LSkuN5tWO6Zy2pZPOWXvUvIkjk0NB7GjTg/cCX7wiOaReDiDmtnp1MzeQs3sLcTnntgZ26EqddhfrQn7qzUjK6Ep+6vlP5pxIKExeTHhvYhakfRvW7/4a4CFKOsZwe3ADcAn3ku/BSaoakQ6jq+0iSBf5mZ3DWHxW65H0CDLVeGyI2NZpFHS5NBTmyzOi13E0Jj5nBmznHjJ5WhsNWIT6xJTrY7PTt3bsZ/wWu2T51epFtJqll1Zh/l08RYmLdjM+ozgbQftJJ1Rsd9zaewM6kloess8pFVI14Zs1kakeX/TtdGx6SxKeS3E8NDFXbmyX+nGZi5rIlgO9FfVA950dWCOXSMIsZzDrkvmIDuYG8PRmCj/IeYeola1eKQC3JClqq5FWrDv+8zLOV7vHGxVa7kqHhN0CfExpb5+VFQiCKSLCcHn/gHveTm74lQJxVUNyZ2jUZ4CPBWn6kFESnfDYLGqAAHed2IqvUASwevAPBH51Ju+GCiHfRIYY4wpjWITgao+KSLfA2fgzgSuVdUloQ7MGGNMeBSaCESklqruE5F6wEbvkT+vnqraEFLGGFMJFHVG8C7wG1wfQ76XqsSbbhPCuIwxxoRJoYlAVX/j/W0dvnCMMcaEW7FtvETk20BeM8YYUzEVdY0gAdfasIGI1OV4k9FaQCm6SjTGGFMeFXVGcCPu+kBH72/+43Pg+UAKF5EhIrJGRFJF5N4ilvudiKiI+L3ZwRhjTOgUdY3gGeAZEbm1NN1JiEgsLmGcB6QDC0RksqquKrBcTeA2YF5J12GMMabsArmP4DkR6Qp0BhJ8Xn+rmLf2AVJVdT2AiLwHjABWFVjun8B44K4SxG2MMSZIArlY/A/gOe9xNm6nPTyAspsCaT7T6d5rvmX3BJqr6pRiYhgjIgtFZGFGRkYAqzbGGBOoQHqG+h1wLrBdVa8FugOBdILjrz+iY/cjiEgM8BTw1+IKUtUJqpqiqikNG4apH3VjjIkSgSSCbFXNA3JEpBawk8BuJksHmvtMNwO2+kzXBLoC34vIRqAfMNkuGBtjTHgF0uncQhGpA/wH12ooC5gfwPsWAO1FpDWwBbgc+H3+TFXdCzTIn/b6M7pLVaOgj2ljjCk/ArlYfLP39CUR+QqoparLA3hfjojcAnwNxAKvqepKERkHLFTVyWUJ3BhjTHAUdUNZclHzVHVxcYWr6hfAFwVeG1vIsoOKK88YY0zwFXVGkD+obQKQAizDXQBOwrX5PyO0oRljjAmHQi8Wq+rZqno2sAlI9lrt9AJ6AqnhCtAYY0xoBdJqqKOq/pQ/oaorgB6hC8kYY0w4BdJqaLWIvAK8g7sP4EpgdUijMsYYEzaBJIJrgZuA273pGcCLIYvIGGNMWAXSfPQQ7g7gp0IfjjHGmHArqvnoB6p6mYj8xIlDVQKgqkkhjcwYY0xYFHVGkF8V9JtwBGKMMSYyihqPYJv3d1P4wjHGGBNuRVUN7cdPlRDupjJV1Vohi8oYY0zYFHVGUDOcgRhjjImMQJqPAiAijThxhLLNIYnIGGNMWAUyQtlwEVkLbAB+ADYCX4Y4LmOMMWESSBcT/8QNGvOLqrbGjVY2K6RRGWOMCZtAEsFRVd0NxIhIjKpOx/oaMsaYSiOQawSZIlID17XERBHZCeSENixjjDHhEsgZwQggG/gL8BWwDhgWyqCMMcaET1H3EfwbeFdVZ/u8/GboQzLGGBNORZ0RrAWeEJGNIvKoiNh1AWOMqYSKGqHsGVXtD5wF7AFeF5HVIjJWRDqELUJjjDEhVew1AlXdpKqPqmpP4PfAb7GBaYwxptII5IayKiIyTEQm4m4k+wW4NOSRGWOMCYuiLhafB4wGLgLmA+8BY1T1QJhiM8YYEwZF3Ufwd+Bd4C5V3ROmeIwxxoRZUb2Pnh3OQIwxxkRGIDeUGWOMqcQsERhjTJSzRGCMMVHOEoExxkQ5SwTGGBPlLBEYY0yUs0RgjDFRzhKBMcZEOUsExhgT5UKaCERkiIisEZFUEbnXz/w7RWSViCwXkW9FpGUo4zHGGHOykCUCEYkFngeGAp2B0SLSucBiS4AUVU0CPgLGhyoeY4wx/oXyjKAPkKqq61X1CK730hG+C6jqdFU96E3OBZqFMB5jjDF+hDIRNAXSfKbTvdcKcz1uvIOTiMgYEVkoIgszMjKCGKIxxphQJgLx85r6XVDkSiAFeMzffFWdoKopqprSsGHDIIZojDGmqPEIyiodaO4z3QzYWnAhERkM3AecpaqHQxiPMcYYP0J5RrAAaC8irUUkHrgcmOy7gIj0BF4GhqvqzhDGYowxphAhSwSqmgPcAnyNG+z+A1VdKSLjRGS4t9hjQA3gQxFZKiKTCynOGGNMiISyaghV/QL4osBrY32eDw7l+o0xxhTP7iw2xpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6KcJQJjjIlylgiMMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6KcJQJjjIlylgiMMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6KcJQJjjIlyIU0EIjJERNaISKqI3OtnflURed+bP09EWoUyHmOMMScLWSIQkVjgeWAo0BkYLSKdCyx2PfCrqrYDngIeDVU8xhhj/AvlGUEfIFVV16vqEeA9YESBZUYAb3rPPwLOFREJYUzGGGMKiAth2U2BNJ/pdKBvYcuoao6I7AXqA7t8FxKRMcAYbzJLRNaUMqYGBcsOEiu3YsUaqnIrUqwVrdyKFGt5LbdlYTNCmQj8HdlrKZZBVScAE8ockMhCVU0pazlWbnjKrGjlVqRYK1q5FSnWilhuKKuG0oHmPtPNgK2FLSMicUBtYE8IYzLGGFNAKBPBAqC9iLQWkXjgcmBygWUmA1d7z38HfKeqJ50RGGOMCZ2QVQ15df63AF8DscBrqrpSRMYBC1V1MvAq8LaIpOLOBC4PVTyeMlcvWblhLbOilVuRYq1o5VakWCtcuWIH4MYYE93szmJjjIlylgiMMSbKRUUiEJHXRGSniKwIcrnNRWS6iKwWkZUicnsQykwQkfkisswr88FgxOpTfqyILBGRKUEsc6OI/CQiS0VkYRDLrSMiH4nIz9533L+M5Z3mxZj/2CcidwQp1r94/68VIjJJRBKCVO7tXpkryxKrv9+AiNQTkW9EZK33t24QyhzpxZonIqVq5lhIuY9528FyEflUROoEqdx/emUuFZGpItIkGOX6zLtLRFREGgQh1gdEZIvP9nthSWMtlKpW+gcwEEgGVgS53FOBZO95TeAXoHMZyxSghve8CjAP6BfEmO8E3gWmBLHMjUCDEPzf3gT+6D2PB+oEsexYYDvQMghlNQU2ANW86Q+Aa4JQbldgBZCIa9gxDWhfyrJO+g0A44F7vef3Ao8GocxOwGnA90BKEGM9H4jznj9a0liLKLeWz/PbgJeCUa73enNcY5lNJf19FBLrA8BdZd2u/D2i4oxAVWcQgvsTVHWbqi72nu8HVuN2CmUpU1U1y5us4j2CckVfRJoBFwGvBKO8UBKRWrgfw6sAqnpEVTODuIpzgXWquilI5cUB1bz7YRI5+Z6Z0ugEzFXVg6qaA/wA/LY0BRXyG/Dt4uVN4OKylqmqq1W1tHf+F1XuVO87AJiLuy8pGOXu85msTil+a0XsX54C7glymSERFYkgHLyeU3vijuDLWlasiCwFdgLfqGqZy/Q8jdsw84JUXj4FporIIq87kGBoA2QAr3tVWa+ISPUglQ2uqfKkYBSkqluAx4HNwDZgr6pODULRK4CBIlJfRBKBCznxJs2yOkVVt4E7qAEaBbHsULoO+DJYhYnIwyKSBlwBjA1SmcOBLaq6LBjl+bjFq8p6raRVeUWxRBAEIlID+Bi4o8ARRqmoaq6q9sAd9fQRka5BiPE3wE5VXVTWsvwYoKrJuJ5m/ywiA4NQZhzu1PhFVe0JHMBVX5SZd4PjcODDIJVXF3d03RpoAlQXkSvLWq6qrsZVg3wDfAUsA3KKfFMlJyL34b6DicEqU1XvU9XmXpm3lLU8L2nfR5CSio8XgbZAD9wBxxPBKtgSQRmJSBVcEpioqp8Es2yvKuR7YEgQihsADBeRjbieYM8RkXeCUC6qutX7uxP4FNfzbFmlA+k+Z0Mf4RJDMAwFFqvqjiCVNxjYoKoZqnoU+AQ4PRgFq+qrqpqsqgNxVQVrg1GuZ4eInArg/d0ZxLKDTkSuBn4DXKFepXmQvQtcGoRy2uIOCpZ5v7dmwGIRaVyWQlV1h3eQmAf8h+D8zgBLBGUiIoKrw16tqk8GqcyG+S0iRKQabifzc1nLVdX/UdVmqtoKVy3ynaqW+ahVRKqLSM3857iLemVunaWq24E0ETnNe+lcYFVZy/WMJkjVQp7NQD8RSfS2iXNx14vKTEQaeX9bAJcQ3Lh9u3i5Gvg8iGUHlYgMAf4GDFfVg0Est73P5HCC81v7SVUbqWor7/eWjmtUsr0s5eYnbc9vCcLv7JhQXIEubw/cj2cbcBT3T7k+SOWegasfXw4s9R4XlrHMJGCJV+YKYGwIvo9BBKnVEK4uf5n3WAncF8Q4ewALve/iM6BuEMpMBHYDtYP8nT6I24msAN4Gqgap3Jm4BLgMOLcM5Zz0G8B1+f4t7izjW6BeEMr8rff8MLAD+DpIsabiuqzP/52VpnWPv3I/9v5ny4H/Ak2DUW6B+Rspeashf7G+DfzkxToZODVY2691MWGMMVHOqoaMMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMBWG19VCfs+L2wv0xBgfYBmv+9ybUNgyfxaRK4IU848issYnzveDUa5P+eml6YnTGF/WfNRUSCLyAJClqo8XeF1w23Ww+1MqFRH5EbhFVZeGqPx0oKsGt0M+E2XsjMBUeCLSzuuz/yVgMXCqiEwQkYVe3/hjfZb9UUR6iEiciGSKyL/Ejf0wx+cu3ofE6/vfW/5f4saIWCMip3uvVxeRj733TvLW1aMEMb8jIi+KyEwR+UVEhnqvVxORN8WN77A4v98mL96nvM+5XERu9inuDq9jvuUi0sFb/hwvtqVeOcHssM9UMpYITGXRGXhVVXuq6w30XlVNAboD54lIZz/vqQ38oKrdgTm4Xi39EVXtA9zN8Y7EbgW2e+/9F67n2cK871M19C+f15sDZwHDgAkiUhXXJ/4RVe0GXAW87VV73YTr0K67qibh+ovKt0Ndx3yv4MabwIt1jLrOCwcCh4qIz0Q5SwSmslinqgt8pkeLyGLcGUInXKIoKFtV87szXgS0KqTsT/wscwbezlhdV8Mri4htlKr28B6+Pah+oKp56vrvTwPae+W+7ZW7EjeuQTtcn1MvqWquN8+3r3p/8c0CnhaRW3GDr+QWEZ+JcpYITGVxIP+J15HY7cA53tHzV4C/oSOP+DzPxXV97c9hP8tImaJ1Cl6g0yLKFT/L5zspPlV9CLgRqAEsKNC5mjEnsERgKqNawH5gn9dj4wUhWMePwGUAItIN/2ccxRkpTgdcNdFaYAZugBREpBNuONRUYCpwk4jEevPqFVWwiLRV1eWq+r+4TgyLbCllolthR0DGVGSLcT12rgDW46pJgu054C0RWe6tbwWwt5Bl3xeRbO/5DlXNT0ypuB1/I1x9/hEReQ54WUR+wvU8+Qfv9ZdxVUfLRSQHN0jJS0XEd5eInIkbjW45LpEY45c1HzWmFMSNTRynqoe8apepuIHlAxpBTNygQB+p6mehjNOYQNgZgTGlUwP41ksIAtwYaBIwpryxMwJjjIlydrHYGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjotz/B8JzJ5H7lKq/AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# Initialize the non-pretrained version of the model used for this run\n", + "scratch_model = model_ft\n", + "scratch_model = scratch_model.to(device)\n", + "scratch_optimizer = optim.SGD(scratch_model.parameters(), lr=0.001, momentum=0.9)\n", + "scratch_criterion = nn.CrossEntropyLoss()\n", + "_,scratch_hist = train_model(scratch_model, dataloaders_dict, scratch_criterion, scratch_optimizer, num_epochs=num_epochs)\n", + "\n", + "# Plot the training curves of validation accuracy vs. number\n", + "# of training epochs for the transfer learning method and\n", + "# the model trained from scratch\n", + "ohist = []\n", + "shist = []\n", + "\n", + "ohist = [h.cpu().numpy() for h in hist]\n", + "shist = [h.cpu().numpy() for h in scratch_hist]\n", + "\n", + "plt.title(\"Validation Accuracy vs. Number of Training Epochs\")\n", + "plt.xlabel(\"Training Epochs\")\n", + "plt.ylabel(\"Validation Accuracy\")\n", + "plt.plot(range(1,num_epochs+1),ohist,label=\"Pretrained\")\n", + "plt.plot(range(1,num_epochs+1),shist,label=\"Scratch\")\n", + "plt.ylim((0,1.))\n", + "plt.xticks(np.arange(1, num_epochs+1, 1.0))\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], "source": [] } @@ -22,25 +515,25 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.7.4" }, "pycharm": { "stem_cell": { "cell_type": "raw", - "source": [], "metadata": { "collapsed": false - } + }, + "source": [] } } }, "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file + "nbformat_minor": 1 +} diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..adffd6d --- /dev/null +++ b/utils.py @@ -0,0 +1,2 @@ +# load images + From 2ebff5276ab089ae171c10b348a7a5701155d770 Mon Sep 17 00:00:00 2001 From: glcanvas Date: Sun, 3 Nov 2019 16:00:26 +0300 Subject: [PATCH 02/10] pytorch sample --- notebooks/init.ipynb | 786 ++++++++++++++++++++++--------------------- utilits.py | 33 ++ utils.py | 2 - 3 files changed, 431 insertions(+), 390 deletions(-) create mode 100644 utilits.py delete mode 100644 utils.py diff --git a/notebooks/init.ipynb b/notebooks/init.ipynb index 7bb39f1..179e2f8 100644 --- a/notebooks/init.ipynb +++ b/notebooks/init.ipynb @@ -2,222 +2,206 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "metadata": { "pycharm": { "is_executing": false, "name": "#%%\n" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "PyTorch Version: 1.2.0\n", - "Torchvision Version: 0.4.0a0\n" - ] - } - ], + "outputs": [], "source": [ "import torch\n", "import torch.nn as nn\n", + "from torch.nn.modules import loss\n", "import torch.optim as optim\n", "import numpy as np\n", "import torchvision\n", + "from torch.optim.optimizer import Optimizer\n", "from torchvision import datasets, models, transforms\n", "import matplotlib.pyplot as plt\n", - "import time\n", - "import os\n", "import copy\n", - "print(\"PyTorch Version: \",torch.__version__)\n", - "print(\"Torchvision Version: \",torchvision.__version__)" + "\n", + "import utilits" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "metadata": { "pycharm": { "is_executing": false, "name": "#%%\n" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "text": [ + "False\n" + ], + "output_type": "stream" + } + ], "source": [ "data_dir = \"/home/nikita/PycharmProjects/hymenoptera_data\"\n", "num_classes = 2\n", "batch_size = 8\n", - "num_epochs = 15\n", - "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")" + "num_epochs = 25\n", + "print(torch.cuda.is_available())\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, + "outputs": [], + "source": [ + "def set_parameter_requires_grad(model, feature_extracting):\n", + " if feature_extracting:\n", + " for param in model.parameters():\n", + " param.requires_grad = False" + ], "metadata": { + "collapsed": false, "pycharm": { - "is_executing": false, - "name": "#%%\n" + "name": "#%%\n", + "is_executing": false } - }, + } + }, + { + "cell_type": "code", + "execution_count": 6, "outputs": [], "source": [ - "def train_model(model, dataloaders, criterion, optimizer, num_epochs=25):\n", - " since = time.time()\n", - "\n", - " val_acc_history = []\n", - "\n", - " best_model_wts = copy.deepcopy(model.state_dict())\n", - " best_acc = 0.0\n", - "\n", - " for epoch in range(num_epochs):\n", - " print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n", - " print('-' * 10)\n", - "\n", - " # Each epoch has a training and validation phase\n", - " for phase in ['train', 'val']:\n", - " if phase == 'train':\n", - " model.train() # Set model to training mode\n", - " else:\n", - " model.eval() # Set model to evaluate mode\n", - "\n", - " running_loss = 0.0\n", - " running_corrects = 0\n", - "\n", - " # Iterate over data.\n", - " for inputs, labels in dataloaders[phase]:\n", - " inputs = inputs.to(device)\n", - " labels = labels.to(device)\n", + "model_ft = models.alexnet(pretrained=True)\n", "\n", - " # zero the parameter gradients\n", - " optimizer.zero_grad()\n", + "set_parameter_requires_grad(model_ft, feature_extracting=False)\n", "\n", - " # forward\n", - " # track history if only in train\n", - " with torch.set_grad_enabled(phase == 'train'):\n", - " # Get model outputs and calculate loss\n", - " # Special case for inception because in training it has an auxiliary output. In train\n", - " # mode we calculate the loss by summing the final output and the auxiliary output\n", - " # but in testing we only consider the final output.\n", - " outputs = model(inputs)\n", - " loss = criterion(outputs, labels)\n", + "params_to_update = []\n", + "for name,param in model_ft.named_parameters():\n", + " if param.requires_grad:\n", + " params_to_update.append(param)\n", "\n", - " _, preds = torch.max(outputs, 1)\n", - "\n", - " # backward + optimize only if in training phase\n", - " if phase == 'train':\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # statistics\n", - " running_loss += loss.item() * inputs.size(0)\n", - " running_corrects += torch.sum(preds == labels.data)\n", - "\n", - " epoch_loss = running_loss / len(dataloaders[phase].dataset)\n", - " epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n", - "\n", - " print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n", - "\n", - " # deep copy the model\n", - " if phase == 'val' and epoch_acc > best_acc:\n", - " best_acc = epoch_acc\n", - " best_model_wts = copy.deepcopy(model.state_dict())\n", - " if phase == 'val':\n", - " val_acc_history.append(epoch_acc)\n", - "\n", - " print()\n", + "optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)\n", + "criterion = nn.CrossEntropyLoss()\n", "\n", - " time_elapsed = time.time() - since\n", - " print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n", - " print('Best val Acc: {:4f}'.format(best_acc))\n", + "# replace last layer from alexNet to custom classification\n", + "num_ftrs = model_ft.classifier[6].in_features\n", + "model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n", "\n", - " # load best model weights\n", - " model.load_state_dict(best_model_wts)\n", - " return model, val_acc_history" - ] + "state_sizes, dataloaders_dict = utilits.image_data(data_dir, batch_size)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": false + } + } }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, + "outputs": [], + "source": [ + "def train_model_single_epoch(model:nn.Module, image_data:dict, criterion:nn.CrossEntropyLoss, optimizer:Optimizer,\n", + " epoch:int, is_grad = True):\n", + " state = 'train'\n", + " model.train()\n", + " for inputs, labels in image_data[state]:\n", + " inputs = inputs.to(device)\n", + " labels = labels.to(device)\n", + " \n", + " # flush evaluated gradients\n", + " # what different ?\n", + " optimizer.zero_grad()\n", + " #model.zero_grad()\n", + " \n", + " output = model(inputs)\n", + " loss = criterion(output, labels)\n", + " if is_grad:\n", + " loss.backward()\n", + " # why bellow command ?\n", + " optimizer.step() \n", + " \n", + " return model" + ], "metadata": { + "collapsed": false, "pycharm": { - "is_executing": false, - "name": "#%%\n" + "name": "#%%\n", + "is_executing": false } - }, + } + }, + { + "cell_type": "code", + "execution_count": 8, "outputs": [], "source": [ - "#image_net = models.alexnet(pretrained=True)\n", - "#print(image_net)\n", - "\n", - "def set_parameter_requires_grad(model, feature_extracting):\n", - " if feature_extracting:\n", - " for param in model.parameters():\n", - " param.requires_grad = False" - ] + "def validate_model_single_epoch(model:nn.Module, image_data:dict, criterion:nn.CrossEntropyLoss, epoch:int):\n", + " state = 'val'\n", + " model.eval()\n", + " sum_loss = 0\n", + " correct_accumulate = 0\n", + " for inputs, labels in image_data[state]:\n", + " inputs = inputs.to(device)\n", + " labels = labels.to(device)\n", + " output = model(inputs)\n", + " loss = criterion(output, labels)\n", + " model_answer = torch.max(output, 1)[1]\n", + " corrects = batch_size - ((model_answer + labels) % 2).sum().item()\n", + " sum_loss += loss.item() * batch_size\n", + " correct_accumulate += corrects\n", + " \n", + " epoch_loss = sum_loss / state_sizes[state]\n", + " epoch_acc = float(correct_accumulate) / state_sizes[state]\n", + " print(\"epoch: {}, corrects:{:.4f}, loss:{:.4f}\".format(epoch, epoch_acc, epoch_loss))\n", + " return epoch_acc" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": false + } + } }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, + "outputs": [], + "source": [ + "def train_model(model, image_data, criterion, optimizer, num_epochs=25, is_grad=True):\n", + " best_model_state = copy.deepcopy(model.state_dict())\n", + " best_acc = 0.0\n", + " for epoch in range(num_epochs):\n", + " print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n", + " print('-' * 10)\n", + " m = train_model_single_epoch(model, image_data, criterion, optimizer, epoch, is_grad)\n", + " acc = validate_model_single_epoch(m, image_data, criterion, epoch)\n", + " if acc > best_acc:\n", + " best_acc = acc\n", + " best_model_state = copy.deepcopy(m.state_dict())\n", + " \n", + " best_model = model.load_state_dict(best_model_state)\n", + " print(best_acc)\n", + " print(best_model)\n", + " return best_model" + ], "metadata": { + "collapsed": false, "pycharm": { - "is_executing": false, - "name": "#%%\n" + "name": "#%%\n", + "is_executing": false } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Initializing Datasets and Dataloaders...\n" - ] - } - ], - "source": [ - "\n", - "\n", - "use_pretrained = False\n", - "feature_extract = True\n", - "model_ft = models.alexnet(pretrained=use_pretrained)\n", - "set_parameter_requires_grad(model_ft, feature_extract)\n", - "num_ftrs = model_ft.classifier[6].in_features\n", - "model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n", - "input_size = 224\n", - "\n", - "# Data augmentation and normalization for training\n", - "# Just normalization for validation\n", - "data_transforms = {\n", - " 'train': transforms.Compose([\n", - " transforms.RandomResizedCrop(input_size),\n", - " transforms.RandomHorizontalFlip(),\n", - " transforms.ToTensor(),\n", - " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n", - " ]),\n", - " 'val': transforms.Compose([\n", - " transforms.Resize(input_size),\n", - " transforms.CenterCrop(input_size),\n", - " transforms.ToTensor(),\n", - " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n", - " ]),\n", - "}\n", - "\n", - "print(\"Initializing Datasets and Dataloaders...\")\n", - "\n", - "# Create training and validation datasets\n", - "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'val']}\n", - "\n", - "# Create training and validation dataloaders\n", - "dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'val']}\n", - "\n", - "# Detect if we have a GPU available\n", - "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n" - ] + } }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": { "pycharm": { "is_executing": false, @@ -227,242 +211,238 @@ "outputs": [ { "name": "stdout", - "output_type": "stream", "text": [ - "Params to learn:\n", - "\t classifier.6.weight\n", - "\t classifier.6.bias\n" - ] + "Epoch 0/24\n----------\n", + "epoch: 0, corrects:0.8497, loss:0.4499\nEpoch 1/24\n----------\n", + "epoch: 1, corrects:0.9216, loss:0.4158\nEpoch 2/24\n----------\n", + "epoch: 2, corrects:0.9150, loss:0.3640\nEpoch 3/24\n----------\n", + "epoch: 3, corrects:0.9020, loss:0.4004\nEpoch 4/24\n----------\n", + "epoch: 4, corrects:0.9216, loss:0.3876\nEpoch 5/24\n----------\n", + "epoch: 5, corrects:0.9477, loss:0.3656\nEpoch 6/24\n----------\n", + "epoch: 6, corrects:0.9085, loss:0.5285\nEpoch 7/24\n----------\n", + "epoch: 7, corrects:0.9477, loss:0.3414\nEpoch 8/24\n----------\n", + "epoch: 8, corrects:0.9412, loss:0.3912\nEpoch 9/24\n----------\n", + "epoch: 9, corrects:0.9281, loss:0.3954\nEpoch 10/24\n----------\n", + "epoch: 10, corrects:0.9281, loss:0.3575\nEpoch 11/24\n----------\n", + "epoch: 11, corrects:0.9608, loss:0.3822\nEpoch 12/24\n----------\n", + "epoch: 12, corrects:0.9412, loss:0.3862\nEpoch 13/24\n----------\n", + "epoch: 13, corrects:0.9346, loss:0.4379\nEpoch 14/24\n----------\n", + "epoch: 14, corrects:0.9150, loss:0.5164\nEpoch 15/24\n----------\n", + "epoch: 15, corrects:0.9281, loss:0.4536\nEpoch 16/24\n----------\n", + "epoch: 16, corrects:0.9281, loss:0.4430\nEpoch 17/24\n----------\n", + "epoch: 17, corrects:0.9412, loss:0.4107\nEpoch 18/24\n----------\n", + "epoch: 18, corrects:0.9477, loss:0.3834\nEpoch 19/24\n----------\n", + "epoch: 19, corrects:0.9477, loss:0.4825\nEpoch 20/24\n----------\n", + "epoch: 20, corrects:0.9216, loss:0.4024\nEpoch 21/24\n----------\n", + "epoch: 21, corrects:0.9020, loss:0.5813\nEpoch 22/24\n----------\n", + "epoch: 22, corrects:0.9346, loss:0.3711\nEpoch 23/24\n----------\n", + "epoch: 23, corrects:0.9477, loss:0.3916\nEpoch 24/24\n----------\n", + "epoch: 24, corrects:0.9412, loss:0.3231\n0.9607843137254902\n\n" + ], + "output_type": "stream" } ], "source": [ - "# Send the model to GPU\n", - "model_ft = model_ft.to(device)\n", - "\n", - "# Gather the parameters to be optimized/updated in this run. If we are\n", - "# finetuning we will be updating all parameters. However, if we are\n", - "# doing feature extract method, we will only update the parameters\n", - "# that we have just initialized, i.e. the parameters with requires_grad\n", - "# is True.\n", - "params_to_update = model_ft.parameters()\n", - "print(\"Params to learn:\")\n", - "if feature_extract:\n", - " params_to_update = []\n", - " for name,param in model_ft.named_parameters():\n", - " if param.requires_grad == True:\n", - " params_to_update.append(param)\n", - " print(\"\\t\",name)\n", - "else:\n", - " for name,param in model_ft.named_parameters():\n", - " if param.requires_grad == True:\n", - " print(\"\\t\",name)\n", - "\n", - "# Observe that all parameters are being optimized\n", - "optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)\n", - "\n" + "trained_model = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 23, "metadata": { "pycharm": { "is_executing": false, "name": "#%%\n" } }, + "outputs": [], + "source": [ + "model_conv = models.alexnet(pretrained=True)\n", + "num_ftrs = model_conv.classifier[6].in_features\n", + "model_conv.classifier[6] = nn.Linear(num_ftrs,num_classes)\n", + "\n", + "params = []\n", + "for param in model_conv.parameters():\n", + " param.requires_grad = False\n", + " params.append(param)\n", + "optimizer_conv = optim.SGD(params, lr=0.001, momentum=0.9)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, "outputs": [ { "name": "stdout", - "output_type": "stream", "text": [ - "Epoch 0/14\n", - "----------\n", - "train Loss: 0.6926 Acc: 0.5451\n", - "val Loss: 0.6925 Acc: 0.6667\n", - "\n", - "Epoch 1/14\n", - "----------\n", - "train Loss: 0.6932 Acc: 0.4877\n", - "val Loss: 0.6928 Acc: 0.4771\n", - "\n", - "Epoch 2/14\n", - "----------\n", - "train Loss: 0.6931 Acc: 0.4795\n", - "val Loss: 0.6927 Acc: 0.5033\n", - "\n", - "Epoch 3/14\n", - "----------\n", - "train Loss: 0.6929 Acc: 0.5123\n", - "val Loss: 0.6930 Acc: 0.4575\n", - "\n", - "Epoch 4/14\n", - "----------\n", - "train Loss: 0.6928 Acc: 0.5082\n", - "val Loss: 0.6928 Acc: 0.4575\n", - "\n", - "Epoch 5/14\n", - "----------\n", - "train Loss: 0.6926 Acc: 0.5451\n", - "val Loss: 0.6932 Acc: 0.4575\n", - "\n", - "Epoch 6/14\n", - "----------\n", - "train Loss: 0.6928 Acc: 0.5246\n", - "val Loss: 0.6932 Acc: 0.4575\n", - "\n", - "Epoch 7/14\n", - "----------\n", - "train Loss: 0.6925 Acc: 0.5246\n", - "val Loss: 0.6925 Acc: 0.4967\n", - "\n", - "Epoch 8/14\n", - "----------\n", - "train Loss: 0.6928 Acc: 0.4836\n", - "val Loss: 0.6932 Acc: 0.4575\n", - "\n", - "Epoch 9/14\n", - "----------\n", - "train Loss: 0.6928 Acc: 0.5205\n", - "val Loss: 0.6929 Acc: 0.4575\n", - "\n", - "Epoch 10/14\n", - "----------\n", - "train Loss: 0.6922 Acc: 0.5533\n", - "val Loss: 0.6931 Acc: 0.4575\n", - "\n", - "Epoch 11/14\n", - "----------\n", - "train Loss: 0.6926 Acc: 0.5410\n", - "val Loss: 0.6933 Acc: 0.4575\n", - "\n", - "Epoch 12/14\n", - "----------\n", - "train Loss: 0.6926 Acc: 0.5205\n", - "val Loss: 0.6939 Acc: 0.4575\n", - "\n", - "Epoch 13/14\n", - "----------\n", - "train Loss: 0.6928 Acc: 0.5082\n", - "val Loss: 0.6935 Acc: 0.4575\n", - "\n", - "Epoch 14/14\n", - "----------\n", - "train Loss: 0.6923 Acc: 0.5246\n", - "val Loss: 0.6930 Acc: 0.4575\n", - "\n", - "Training complete in 1m 28s\n", - "Best val Acc: 0.666667\n" - ] + "Epoch 0/14\n----------\n", + "epoch: 0, corrects:0.5752, loss:0.7454\nEpoch 1/14\n----------\n", + "epoch: 1, corrects:0.5752, loss:0.6931\nEpoch 2/14\n----------\n", + "epoch: 2, corrects:0.5752, loss:0.7215\nEpoch 3/14\n----------\n", + "epoch: 3, corrects:0.5752, loss:0.6915\nEpoch 4/14\n----------\n", + "epoch: 4, corrects:0.5752, loss:0.6903\nEpoch 5/14\n----------\n", + "epoch: 5, corrects:0.5752, loss:0.7229\nEpoch 6/14\n----------\n", + "epoch: 6, corrects:0.5752, loss:0.6956\nEpoch 7/14\n----------\n", + "epoch: 7, corrects:0.5752, loss:0.6935\nEpoch 8/14\n----------\n", + "epoch: 8, corrects:0.5752, loss:0.7454\nEpoch 9/14\n----------\n", + "epoch: 9, corrects:0.5752, loss:0.7066\nEpoch 10/14\n----------\n", + "epoch: 10, corrects:0.5752, loss:0.6941\nEpoch 11/14\n----------\n", + "epoch: 11, corrects:0.5752, loss:0.7106\nEpoch 12/14\n----------\n", + "epoch: 12, corrects:0.5752, loss:0.7062\nEpoch 13/14\n----------\n", + "epoch: 13, corrects:0.5752, loss:0.7438\nEpoch 14/14\n----------\n", + "epoch: 14, corrects:0.5752, loss:0.7209\n0.5751633986928104\n\n" + ], + "output_type": "stream" } ], "source": [ - "# Setup the loss fxn\n", - "criterion = nn.CrossEntropyLoss()\n", + "trained_model_conv = train_model(model_conv, dataloaders_dict, criterion, optimizer_conv, num_epochs=15, is_grad=False)\n", "\n", - "# Train and evaluate\n", - "model_ft, hist = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft, num_epochs=num_epochs)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, + "\n", + "\n", + "\"\"\"\"\n", + "\n", + "d) \n", + "H =\n", + "1 1 1 1 1 1\n", + "0 1 0 1 1 0\n", + "0 0 1 0 1 1\n", + "=>\n", + "H = \n", + "1 1 0 1 0 0\n", + "1 0 0 0 1 0\n", + "1 0 1 0 0 1\n", + "G = \n", + "1 0 0 1 1 1\n", + "0 1 0 1 0 0\n", + "0 0 1 0 0 1\n", + "=> r = 3, k = 3, n = 6, d = 1\n", + " \n", + "e) \n", + "G = \n", + "0 0 1 0 1 1\n", + "1 1 1 1 1 1\n", + "1 0 1 1 0 0\n", + "=> \n", + "G = \n", + "0 0 1 1 0 0\n", + "1 1 1 0 1 0\n", + "1 1 1 0 0 1\n", + "H = \n", + "0 0 1 1 0 0\n", + "1 1 1 0 1 0\n", + "1 1 1 0 0 1\n", + "n = 6, k = 3, r = 3, d = 1\n", + "\n", + "f) \n", + "G = \n", + "0 1 0 1 1 0\n", + "1 1 1 1 1 1 \n", + "=> \n", + "G = \n", + "1 0 1 0 0 1\n", + "0 1 0 1 1 0\n", + "H = \n", + "1 0 1 0 0 0 \n", + "0 1 0 1 0 0\n", + "0 1 0 0 1 0\n", + "1 0 0 0 0 1\n", + " ^ ^ ^\n", + "=>\n", + "n = 6, k =2, r = 4, d = 2 \n", + "g)\n", + "H = \n", + "0 1 0 1 1 0 0\n", + "1 0 1 1 0 1 0\n", + "1 1 1 1 1 1 1\n", + " ^ ^\n", + "=> \n", + "H =\n", + "0 1 0 1 1 0 0\n", + "1 0 1 1 0 1 0\n", + "0 0 0 1 0 0 1\n", + "=>\n", + "G =\n", + "1 0 0 0 0 1 0\n", + "0 1 0 0 1 0 0\n", + "0 0 1 0 0 1 0\n", + "0 0 0 1 1 1 1\n", + "=>\n", + "n = 7, k = 4, r = 3, d = 1\n", + "\n", + "h)\n", + "G = \n", + "1 1 1 0 1 0 0\n", + "0 1 1 1 0 1 0\n", + "0 0 1 1 1 0 1\n", + "=> \n", + "G = \n", + "1 0 0 1 1 1 0\n", + "0 1 0 0 1 1 1\n", + "0 0 1 1 1 0 1\n", + "=>\n", + "H = \n", + "1 0 1 1 0 0 0\n", + "1 1 1 0 1 0 0\n", + "1 1 0 0 0 1 0\n", + "0 1 1 0 0 0 1\n", + "^ ^ ^ ^\n", + "=> \n", + "n = 7, k = 3, r = 4, d = 3\n", + "\n", + "i)\n", + "H = \n", + "1 1 1 0 1 0 0\n", + "0 1 1 1 0 1 0\n", + "1 1 1 1 1 1 1\n", + "^ ^\n", + "=> \n", + "H = \n", + "1 1 1 0 1 0 0 \n", + "0 1 1 1 0 1 0\n", + "0 1 1 0 0 0 1\n", + "=> \n", + "G = \n", + "1 0 0 1 0 0 0\n", + "1 1 1 0 1 0 0\n", + "1 1 1 0 0 1 0\n", + "0 1 0 0 0 0 1\n", + "=> \n", + "n = 7, k = 4, r = 3, d = 1\n", + "\n", + "\"\"\"" + ], "metadata": { + "collapsed": false, "pycharm": { - "is_executing": false, - "name": "#%%\n" + "name": "#%%\n", + "is_executing": false } - }, + } + }, + { + "cell_type": "code", + "execution_count": 80, "outputs": [ { "name": "stdout", - "output_type": "stream", "text": [ - "Epoch 0/14\n", - "----------\n", - "train Loss: 0.6926 Acc: 0.4918\n", - "val Loss: 0.6923 Acc: 0.6340\n", - "\n", - "Epoch 1/14\n", - "----------\n", - "train Loss: 0.6926 Acc: 0.5533\n", - "val Loss: 0.6927 Acc: 0.5033\n", - "\n", - "Epoch 2/14\n", - "----------\n", - "train Loss: 0.6932 Acc: 0.5164\n", - "val Loss: 0.6930 Acc: 0.4575\n", - "\n", - "Epoch 3/14\n", - "----------\n", - "train Loss: 0.6928 Acc: 0.5287\n", - "val Loss: 0.6931 Acc: 0.4575\n", - "\n", - "Epoch 4/14\n", - "----------\n", - "train Loss: 0.6927 Acc: 0.5000\n", - "val Loss: 0.6921 Acc: 0.6928\n", - "\n", - "Epoch 5/14\n", - "----------\n", - "train Loss: 0.6924 Acc: 0.5410\n", - "val Loss: 0.6926 Acc: 0.4902\n", - "\n", - "Epoch 6/14\n", - "----------\n", - "train Loss: 0.6921 Acc: 0.5410\n", - "val Loss: 0.6928 Acc: 0.4575\n", - "\n", - "Epoch 7/14\n", - "----------\n", - "train Loss: 0.6922 Acc: 0.5902\n", - "val Loss: 0.6926 Acc: 0.4641\n", - "\n", - "Epoch 8/14\n", - "----------\n", - "train Loss: 0.6922 Acc: 0.5533\n", - "val Loss: 0.6924 Acc: 0.4967\n", - "\n", - "Epoch 9/14\n", - "----------\n", - "train Loss: 0.6928 Acc: 0.5082\n", - "val Loss: 0.6931 Acc: 0.4575\n", - "\n", - "Epoch 10/14\n", - "----------\n", - "train Loss: 0.6929 Acc: 0.5410\n", - "val Loss: 0.6931 Acc: 0.4575\n", - "\n", - "Epoch 11/14\n", - "----------\n", - "train Loss: 0.6925 Acc: 0.5943\n", - "val Loss: 0.6925 Acc: 0.4706\n", - "\n", - "Epoch 12/14\n", - "----------\n", - "train Loss: 0.6922 Acc: 0.5041\n", - "val Loss: 0.6915 Acc: 0.6993\n", - "\n", - "Epoch 13/14\n", - "----------\n", - "train Loss: 0.6924 Acc: 0.5287\n", - "val Loss: 0.6924 Acc: 0.4837\n", - "\n", - "Epoch 14/14\n", - "----------\n", - "train Loss: 0.6921 Acc: 0.5779\n", - "val Loss: 0.6923 Acc: 0.4706\n", - "\n", - "Training complete in 1m 31s\n", - "Best val Acc: 0.699346\n" - ] + "[0.5691011400000001, 0.6513124383000002, 0.7175702976840004, 0.7712320727218503, 0.8146979798789038, 0.8499053641763854, 0.8784233453999499, 0.9015229097775138, 0.9202335569213986, 0.9353891811077326, 0.9476652366972641, 0.9576088417247846, 0.9656631617970758, 0.9721871610556316, 0.9774716004550617, 0.9817519963686003, 0.9852191170585665]\n[0.0772553001108834, 0.09561792499002673, 0.11361512828387049, 0.1312541872310217, 0.14854222890512433, 0.16548623854991237, 0.18209306240276912, 0.198369410460954, 0.214321859192781, 0.22995685419484466, 0.24528071279636723, 0.2602996266117195, 0.2750196640421463, 0.2894467727277076, 0.3035867819504261, 0.3174454049896126, 0.3310282414303194]\n[0.007972055930000112, 0.009955119790251791, 0.011934219505791075, 0.013909363000999002, 0.015880558184360002, 0.017847812948549473, 0.019811135170465316, 0.021770532711259557, 0.023726013416369752, 0.025677585115550426, 0.02762525562290444, 0.02956903273691425, 0.03150892424047316, 0.03344493790091646, 0.03537708147005253, 0.037305362684193884, 0.039229789264188186]\n" + ], + "output_type": "stream" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXgUVdb48e9JQghhX0X2XdYAIWyiiIoKOoCjg8io4zbi6LiNo47v+JNRRn1H3HXcGHdF3BeG1wVRFGTfkUUkrAlrAAMEwpLk/P64FWhCJ+kkvSTp83meftLVVX3rdKe6TtWtW/eKqmKMMSZ6xUQ6AGOMMZFlicAYY6KcJQJjjIlylgiMMSbKWSIwxpgoZ4nAGGOinCWCAIhIKxFREYnzpr8UkasDWbYU6/q7iLxSlnhNxVfW7SgI6x8gImtFJEtELg7xumK99bQI5rIVgYi8IyIPRDqOqEgEIvK1iIzz8/oIEdle0h+bqg5V1TeDENcgEUkvUPYjqvrHspZdzDpVRO4J1ToqIxG5xvve7i7werqIDIpQWKE0Dvi3qtZQ1c98Z3g74vxHnohk+0xfUdIVqWqut57NwVy2pETkIRE5WuDz7Qr2esqjqEgEwBvAVSIiBV6/CpioqjnhDylirgb2eH/DKlJHt0G0B/ibiNSKdCAlUcrvvSWw0t8Mb0dcQ1VrAJuBYT6vTQzS+iNlou/nU9UGkQ4oHKIlEXwG1APOzH9BROoCvwHe8qYvEpElIrJPRNKKOl0Tke9F5I/e81gReVxEdonIeuCiAsteKyKrRWS/iKwXkRu916sDXwJNfI4+mojIAyLyjs/7h4vIShHJ9NbbyWfeRhG5S0SWi8heEXlfRBKKiDsR+B3wZ6C9iKQUmH+GiMz21pUmItd4r1cTkSdEZJO3nh+91046o/FiGuw9f0BEPvJOf/cB14hIHxGZ461jm4j8W0Tifd7fRUS+EZE9IrLDqyprLCIHRaS+z3K9RCRDRKoUWH8T7wi1ns9rPb3/TxURaSciP3ifY5eIvF/Y9+XHamAO8JdCvt83ROQhn+kTvh/vu7nb+38dEJFXReQUcVWN+0Vkmrdd+rpORLZ639VffcqKEZF7RWSdiOwWkQ/yP7Mcr1a6XkQ2A98VEu8NIpLqfdeTRaSJ9/o6oA3wX2+7rFqC7yj/yPp9EZkkIvuBK0Wkv4jM9fm/P5v/vxOROC/eVt70O978/O9ljoi0Lumy3vyhIvKL9/9+TkRm5W/XJfxM+eu9VUQ2eNvOv0QkxpsfIyJjvd/ITm9bqOXz/oHe598r7rd1lU/x9Qr5rDHeZ9vpvW+5iHQuaewBUdWoeAD/AV7xmb4RWOozPQjohkuOScAO4GJvXitAgThv+nvgj97zPwE/A81xyWZ6gWUvAtoCApwFHASSfdaZXiDOB4B3vOcdgAPAeUAV4B4gFYj35m8E5gNNvHWvBv5UxHdwFbANiAX+CzzrM68FsB8Y7a2rPtDDm/e895mbeu89HahaSPwbgcE+n+UocLH3vVYDegH9gDjve10N3OEtX9OL769Agjfd15v3BXCTz3qeAp4r5HN+B9zgM/0Y8JL3fBJwnxdPAnBGgNvPNcCPQA8gE6jnvZ4ODPKevwE8VGCbSi/w3cwFTvG+y53AYqCn931+B/yjwDY3CaiO2zYzfL7bO7yymnnvfRmYVOC9b3nvrebn85wD7AKSvfc/B8zw938s5ns5aTngIeAIMMzn/94b6Ov939sAvwC3eMvHefG28qbf8WJLwW2L73P8N1GSZRvhtukR3rw7cdvjNYV8loeANwqZl7/eaUBd7ztOzS8LGON9pta47fZz4HVvXmsvjsu8chpw/LdVVPwX4X7ftb3vsTPQOCT7x1AUWh4fwBnA3vwfBTAL+EsRyz8NPFXgh+UvEXyHz84XON93WT/lfgbc7j0fRNGJ4H7gA595McAWju94NgJX+swfj7fDK2Td04CnveejcTuWKt70/wCf+nlPDJANdPczz1/8GzkxEcwoLB5vmTvy1+vFtKSQ5UYBs7znscB2oE8hy/4R+M57LkAaMNCbfguYADQr4fZzDfCj9/wD4FHveUkTwRU+0x8DL/pM3wp8VmCb61jg//uq93w1cK7PvFNxO7k4n/e2KeLzvAqM95mu4b2/VcH/YzHfy0nL4Xao3xXzvruAD73n/nbuL/ksOxxYUYplrwNm+swT3IHGNYXElJ/AMn0e3xRY72Cf5W8Dvvae/wCM8ZnXBTiM+/3cn/9Z/ayzqPjPxx1k9gViSrK9lvQRLVVDqOqPuB3fCBFpgztCeTd/voj0FZHpXnXDXtyRfiD1g01wO5p8m3xneqemc73T70zgwgDLzS/7WHmqmuetq6nPMtt9nh/E/aBPIiLNgbOB/Drcz3FHxPlVWc2BdX7e2sBbzt+8QPh+N4hIBxGZIu4i/T7gEY5/H4XFkB9vZ+9/dx6wV1XnF7LsR0B/r6pjIO4HPNObdw9uhzBfXJXbdaX4TGOBm0SkcSneu8Pnebaf6YL/v4LbVhPveUvgU6+qJROXGHJxZxv+3ltQwW0rC9jNidtWWRT8v3cUkf/z+b+Po+jfQUDbdTHLnvDbVLd3PaEq0493VbWOz+O8AvML+3+c8H16z+OBhhS9XRcav6pOBV4CXgR2iMhLIlKzmPhLJWoSgect4A+4KpKpqur7I3wXmAw0V9XauH9AwYvL/mzD/aPzHWvW5tWtfgw8DpyiqnVwVRz55WoxZW/F/eDzyxNvXVsCiKugq3D/7/+KyHZgPW4H/wdvfhquCqugXcChQuYdABJ94ovFbfi+Cn7GF3FHOe1VtRbwd45/H4XFgKoewh2JX+F9lrf9LectmwlMxZ2K/x5XZaLevO2qeoOqNsFVD74gIu0KK6uQ8n8GPvFi93XC9wGUJlEUVHDb2uo9TwOGFthpJaiq77ZR1PZVcNuqjqsOLM225U/Bdb8MrADaef/3sQT2+yqLbbiqM+DY76esia6w/8cJ36c37wju4LPQ7bo4qvq0qiYDXXFVQ3eWppziRGMiGAzcABRs/lkT2KOqh0SkD24HEogPgNtEpJl3oe9en3nxuPrXDCBHRIbiTvfy7QDqi0jtIsq+SETO9S6s/RV3ujk7wNh8/QF4EFfHnf+41Cu/Pu5MYbCIXOZdGKsvIj28s5DXgCfFXYiN9S78VcXViSaIu9BeBfh/3uctSk1gH5AlIh2Bm3zmTQEai8gdIlJVRGqKSF+f+W/hqmiG406pi/Ku95kv5cQzv5Eikr9z+BW3w8otpix/HgSuBer4vLYUuFBE6nlnC3eUotyC7heRRBHp4q0v/+L2S8DDItISQEQaisiIEpT7LnCtiPTw/pePAPNUdWMQYvanJq5q9oC4Bg83hmg9vqYAySIyTFzLpds5+UClpO4RkTri7mO4jeP/j0nAneIu1NcEHsYdgOThttUhInKp99tqICLdi1uRuIYVfbzYD+ASS2m21WJFVSLwNvLZuAtokwvMvhkYJ66Vw1jcTjgQ/wG+BpbhLvx94rO+/biN5QPcTuf3vuv1jiwnAeu9U/wmPuWiqmuAK3EX8nbhLr4NU9UjAcYGgIj0w9UbP+8dEec/JuMueI1W1y77Qlyy2YPbqeVvrHcBPwELvHmP4uos9+K+t1dwR5IHKP7U+y7ve9iP++6Otdrxvq/zvM+5HViLq87Knz8LyAMWB7DDmgy0B3ao6jKf13sD80Qky1vmdlXd4H1PKyXAdvDee97GbUv53sZtBxtxZyQlaZFUmB9w/6Nvgce96gKAZ7z4p3rb7FxcXXJAVPVbXN31x7gj57bA5UGItzB/xTVZ3o87OwjGd1Mk74x/FPAkrtqrLbAEdzBVmCvkxPsIssSntRqukcVSr5xPcdeF4Pi2PBN3tr0fl3jyt5VhwN9wv5/FuIv/xamDu5aTidumtuEaSQSdeGfMxlQIIvIdrh7X7r42JeJVXW4FfqeqM4tbvsB743AX01uH8KwpYqLqjMBUbCLSG9fcMeRHk6ZyEJEhIlLbq/66H8jBNck0PkKWCETkNe9GiBWFzBfvZolU70aJ5FDFYio+EXkT1/z1Dq8KyZhAnIGrqtkFDMHdG1RU1VBUClnVkIgMBLKAt1S1q5/5F+LaTV+Iq9t8RlUDruM0xhgTHCE7I1DVGbgLI4UZgUsSqqpzgToicmqo4jHGGONfJDuDasqJN2eke69tK7igiIzB3cJN9erVe3Xs2DEsARpjTGWxaNGiXarqt/lsJBOBv5tJ/NZTqeoEXLcApKSk6MKFC0MZlzHGVDoisqmweZFsNZTOiXfpNeP4XXrGGGPCJJKJYDLwB6/1UD9c3zEnVQsZY4wJrZBVDYnIJFzviw3E9cn+D1w3q6jqS7g+dy7E3TV5EHf7vDHGmDALWSJQ1dHFzFfcACnGmChz9OhR0tPTOXToUKRDqXQSEhJo1qwZVapUKX5hT0UaQs4YU0mkp6dTs2ZNWrVqhZw0gqwpLVVl9+7dpKen07p16+Lf4LEuJowxYXfo0CHq169vSSDIRIT69euX+EzLEoExJiIsCYRGab5XSwTGGBPlLBEYY6JSbGwsPXr0oGvXrowcOZKDBw+W6P1PP/10id8DMHbsWKZNm1bi9/kzaNAggnGDrSUCY0xUqlatGkuXLmXFihXEx8fz0ksvnTBfVcnLyyv0/UUlgtzcwgcSGzduHIMHDy5d0CFiicAYE/XOPPNMUlNT2bhxI506deLmm28mOTmZtLQ0pk6dSv/+/UlOTmbkyJFkZWXx7LPPsnXrVs4++2zOPtsNolejRg3Gjh1L3759mTNnDuPGjaN379507dqVMWPGkN/T8zXXXMNHH30EQKtWrfjHP/5BcnIy3bp14+effwbgwIEDXHfddfTu3ZuePXvy+eefA5Cdnc3ll19OUlISo0aNIjs7Oyif35qPGmMi6sH/rmTV1n1BLbNzk1r8Y1iXgJbNycnhyy+/ZMiQIQCsWbOG119/nRdeeIFdu3bx0EMPMW3aNKpXr86jjz7Kk08+ydixY3nyySeZPn06DRo0ANzOu2vXrowbN87F0LkzY8eOBeCqq65iypQpDBs27KT1N2jQgMWLF/PCCy/w+OOP88orr/Dwww9zzjnn8Nprr5GZmUmfPn0YPHgwL7/8MomJiSxfvpzly5eTnBycYVwsERhjolJ2djY9evQA3BnB9ddfz9atW2nZsiX9+vUDYO7cuaxatYoBAwYAcOTIEfr37++3vNjYWC699NJj09OnT2f8+PEcPHiQPXv20KVLF7+J4JJLLgGgV69efPKJG/J86tSpTJ48mccffxxwzW03b97MjBkzuO222wBISkoiKSkpGF+FJQJjTGQFeuQebPnXCAqqXr36seeqynnnncekSZOKLS8hIYHY2FjA7bhvvvlmFi5cSPPmzXnggQcKbdtftWpVwCWSnJycY+v9+OOPOe20005aPhTNbu0agTHGFKJfv37MmjWL1NRUAA4ePMgvv/wCQM2aNdm/3/+oqfk7/QYNGpCVlXXsmkCgLrjgAp577rlj1xWWLFkCwMCBA5k4cSIAK1asYPny5SX/UH5YIjDGmEI0bNiQN954g9GjR5OUlES/fv2OXdAdM2YMQ4cOPXax2FedOnW44YYb6NatGxdffDG9e/cu0Xrvv/9+jh49SlJSEl27duX+++8H4KabbiIrK4ukpCTGjx9Pnz59yv4hCeGYxaFiA9MYU/GtXr2aTp06RTqMSsvf9ysii1Q1xd/ydkZgjDFRzhKBMcZEOUsExhgT5SwRGGNMlLNEYIwxUc4SgTHGRDlLBMaYqPTwww/TpUsXkpKS6NGjB/PmzStTeZmZmbzwwgvFLhesrqODyRKBMSbqzJkzhylTprB48WKWL1/OtGnTaN68ebHvy+8Cwp9AE0F5ZInAGBN1tm3bRoMGDY7189OgQQOaNGnCggULOP300+nevTt9+vRh//79vPHGG4wcOZJhw4Zx/vnnk5WVxbnnnnus6+j8LqLvvfde1q1bR48ePbj77rsBGD9+PN26daN79+7ce++9x9b/4Ycf0qdPHzp06MDMmTPD/wUUYJ3OGWMi68t7YftPwS2zcTcY+q9CZ59//vmMGzeODh06MHjwYEaNGkX//v0ZNWoU77//Pr1792bfvn1Uq1YNcGcQy5cvp169euTk5PDpp59Sq1Ytdu3aRb9+/Rg+fDj/+te/WLFixbGO7L788ks+++wz5s2bR2JiInv27Dm2/pycHObPn88XX3zBgw8+GLQRy0rLEoExJurUqFGDRYsWMXPmTKZPn86oUaO47777OPXUU4/1C1SrVq1jy5933nnUq1cPcD2D/v3vf2fGjBnExMSwZcsWduzYcdI6pk2bxrXXXktiYiLAsffDiV1Pb9y4MVQfM2CWCIwxkVXEkXsoxcbGMmjQIAYNGkS3bt14/vnnC+3i2bdr6okTJ5KRkcGiRYuoUqUKrVq18tvFtKoWWp6/rqcjya4RGGOizpo1a1i7du2x6aVLl9KpUye2bt3KggULANi/f7/fnfTevXtp1KgRVapUYfr06WzatAk4uVvq888/n9dee+3YuMa+VUPljZ0RGGOiTlZWFrfeeiuZmZnExcXRrl07JkyYwLXXXsutt95KdnY21apV81t3f8UVVzBs2DBSUlLo0aMHHTt2BKB+/foMGDCArl27MnToUB577DGWLl1KSkoK8fHxXHjhhTzyyCPh/qgBsW6ojTFhZ91Qh5Z1Q22MMaZELBEYY0yUs0RgjImIilYtXVGU5nu1RGCMCbuEhAR2795tySDIVJXdu3eTkJBQovdZqyFjTNg1a9aM9PR0MjIyIh1KpZOQkECzZs1K9B5LBMaYsKtSpQqtW7eOdBjGY1VDxhgT5UKaCERkiIisEZFUEbnXz/wWIjJdRJaIyHIRuTCU8RhjjDlZyBKBiMQCzwNDgc7AaBHpXGCx/wd8oKo9gcuBitmZtzHGVGChPCPoA6Sq6npVPQK8B4wosIwC+V381Qa2hjAeY4wxfoQyETQF0nym073XfD0AXCki6cAXwK3+ChKRMSKyUEQWWisDY4wJrlAmAn/9rxZsNDwaeENVmwEXAm+LyEkxqeoEVU1R1ZSGDRuGIFRjjIleoUwE6YDvIKDNOLnq53rgAwBVnQMkAA1CGJMxxpgCQpkIFgDtRaS1iMTjLgZPLrDMZuBcABHphEsEVvdjjDFhFLJEoKo5wC3A18BqXOuglSIyTkSGe4v9FbhBRJYBk4Br1O45N8aYsArpncWq+gXuIrDva2N9nq8CBoQyBmOMMUWzO4uNMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6JcsYnAG3LSGGNMJRXIGUGqiDzmZ7xhY4wxlUAgiSAJ+AV4RUTmesNG1iruTcYYYyqGYhOBqu5X1f+o6unAPcA/gG0i8qaItAt5hMYYY0IqoGsEIjJcRD4FngGeANoA/6XAWAPGGFNh/PAYvHQG5B6NdCQRF0jV0FpgBPCYqvZU1SdVdYeqfgR8FdrwTIWQsQbmvAA2uJypKA5nweznYPtPsPLTSEcTcYGMUJakqln+ZqjqbUGOx1REX98Hqd/AKV2gzVmRjsaY4i2bBIf3QmJ9mPUMdBsJIpGOKmICOSN4XkTq5E+ISF0ReS2EMZmKZPc6lwQAfhgf2ViMCUReHsx9EZr2gvP+CTtWQOq0SEcVUQG1GlLVzPwJVf0V6Bm6kEyFMv8/EFMFzrgTNv0IG3+MdETGFC31G9izDvrd7M4EajV1ZwVRLJBEECMidfMnRKQeIR703lQQh7Ng6UToPALOugdqnAI/PBrpqIwp2twXoeapbruNi3cJYeNMSF8U6cgiJpBE8AQwW0T+KSL/BGYDFbIOIPtIbqRDqFyWvweH90HfG6FKNRhwO2yYAZvmRDoyY/zbuRrWT4c+N0BsFfdar6shoTbMeiqysUVQIPcRvAX8DtgB7AQuUdW3Qx1YsL05eyODHp9O1uGcSIdSOai6aqFTe0Cz3u61XtdC9YZ2VmDKr7kvQlyC21bzVa0Jvf8Iq6fArtTIxRZBAXU6p6orgQ+Az4EsEWkR0qhCIKlZbXbsO8xrP26IdCiVw4YfIONndzaQ39oiPhFOv80dcaXNj2x8xhR0YDcsfx+6Xw6J9U6c1/dPEBsPs5+NTGwRFsgNZcNFZC2wAfgB2Ah8GeK4gq5ni7qc3/kU/jNjPb8eOBLpcCq+eRNc07sul5z4esp17nVrQWTKm0WvQ84h6HvTyfNqNIIev3fNSvdvD39sERbIGcE/gX7AL6raGjgXmBXSqELkrgtOI+tIDi/+sC7SoVRsv26CX76E5KuhSsKJ86rWgP63uJYZUXzxzZQzOUdgwSvQ5mxo1NH/MqffCnk5MO+l8MZWDgSSCI6q6m5c66EYVZ0O9AhxXCHR4ZSaXNKzGW/O3sj2vYciHU7FtfBVQKD39f7n97kBqtWFGXZWYMqJVZ/D/m2uhVBh6reFTsNhwWtwaF/4YisHAkkEmSJSA5gBTBSRZ4AKe8X1jsHtyVPlmW/XRjqUiuloNix+CzpeBLWb+V+mak3o/2f45SvYujS88RlTkCrMfR7qt4N2g4tedsDt7o7jRa+HJ7ZyIpBEMAI4CPwF17fQOmBYKIMKmbxcmtdL5Iq+LflgYRobdh2IdEQVz08fQvav7iJxUfqMcU3y7FqBibS0+bB1ibsgHFPMLq9pMrQe6FoX5RwOT3zlQJHfijc62eeqmqeqOar6pqo+61UVVSzLP4CXz4LD+/nz2e2oGhfDk9/8EumoKhZVmD8BGnWGlgOKXjahNvT7M6z5P9i2PDzxGePPvBfd9th9dGDLD7jDVSMt/yC0cZUjRSYCVc0FDopI7TDFEzq1msLOVTD5VhrWiOe6Aa3577KtrNy6N9KRVRyb57reGvuMCayDrr43QtVaMOOx0MdmjD+ZabBqsmvYULVGYO9pew407ua6ncjLC2185UQgVUOHgJ9E5FUReTb/EerAgq7VADj3ftfl7PwJ3DCwDbWrVeHxr9dEOrKKY/7L7sgq6bLAlq9Wx52Or54MO1aGNjZj/FnwH/e3z5jA3yPizgp2r3Wt46JAIIng/4D7cReLF/k8Kp7Tb4cOQ+Hr+6i9exk3DWrL9DUZLNi4J9KRlX/7trojq55XQXz1wN/X7yaIr2lnBSb8jhyARW9Ap2FQp3nJ3tv5YqjTAn58OirG2Qiki4k3/T3CEVzQxcTAb1+EWqfCB1dzdfdaNKpZlfFf/YxGwT+7TBa+DprnbsUvicR60HcMrPwMdv4cmtiM8WfZJDi01x2MlFRsHPS/FdLnw+bK33dWIHcWbxCR9QUfgRQuIkNEZI2IpIrIvYUsc5mIrBKRlSLybkk/QIlVqwuXvQUHdlJtyp+47Zy2LNj4K9+vyQj5qiusnMOuOV2HC6Be65K/v9+foUqinRWY8MnLg7kvQZOe0Lxv6croeeXxgWsquUCqhlKA3t7jTOBZ4J3i3uS1OHoeGAp0BkaLSOcCy7QH/gcYoKpdgDtKFH1pNekJQx+F1GmMPvwhLesnMv7rNeTl2VmBXys/gwMZJatn9VW9vrvJbMXHkGEttUwYrPvW1fH3u7n0I4/FJ0KfG939MDtWBTe+ciaQqqHdPo8tqvo0cE4AZfcBUlV1vaoeAd7D3ZPg6wbgeW+wG1R1ZwnjL71e10K3y4j9/hH+t/suVm/bx5SftoVt9RXK/Jehfnt3e35p9b/FdVU984ngxWVMYea+ADUau7r+suhzgzubreSd0QVSNZTs80gRkT8BNQMouymQ5jOd7r3mqwPQQURmichcERlSSAxjRGShiCzMyAhSFY4I/OYpaHga/Zfey4BGR3hy6hqO5kZHc7GApS+CLYvc2UBxN+MUpUZD1yHdTx+44S2NCZWdP8O676DPH93AM2WRWA+S/+BupNybHpz4yqFAB6bJf/wvkAwE0n7Q3/lYwbqXOKA9MAgYDbziOz7ysTepTlDVFFVNadiwYQCrDlDVGnDZW8jRbJ6Pf4703fv4cGHl/WeXyvwJrtVPjwBvxinKgNtdV792VmBCad5LJ485UBb9/+xaDs15ITjllUOBVA2d7fM4T1XHqGogje/TAd82W82ArX6W+VxVj6rqBmANLjGET8PTYPiz1Nm1iCfqfc4z3/7CoaM2khkAWRmw8hOXBKoGchJYjBqN3FnBsvdgj40LYULg4B63fSVdBtUbBKfMOi2g66WuKerBytnUPJCqoUd8j9JFpK6IPBRA2QuA9iLSWkTigcuByQWW+Qw42yu3Aa6qKKAWSUHV7XfQ+wZGHPyYHlk/8tacjWEPoVxa9AbkHin9RWJ/Tr8NYuLsrMCExqI3ICfb/5gDZTHgdjh6wOt5t/IJpGpoqKpm5k94F3YvLO5NqpoD3AJ8DawGPlDVlSIyTkSGe4t9DewWkVXAdODuiPVjdMHD0CSZp6q+zOTps9h36GhEwig3co+6jb7tOdAgiCdptU51Y8Qum+TGNTAmWHKPuuFTW58Fp3QufvmSaNwV2p3nmqQezQ5u2eVAIIkgVkSq5k+ISDWgahHLH6OqX6hqB1Vtq6oPe6+NVdXJ3nNV1TtVtbOqdlPV90rzIYIiriqMfIP4+Co8mvs4b0yv3M3FivXzFNfxVp9iehktjQF3gMTAj9E7WLgJgVWfw/6tRY85UBYDboeDu2DpxNCUH0GBJIJ3gG9F5HoRuQ74BqiYdxYXp25L4i79D11iNtF47oPsyoqebmhPMm8C1G0F7c8Lftm1m7quKpa84zoFMyYY5r4I9dpA+/NDU36rM6BpL5j9HORVruuIgVwsHg88BHQCugD/9F6rnDpcwK+9buUy+ZaZH1butsOF2v4TbJ7tupOIiQ3NOs74i/s76+nQlG+iS9oC2LLQXRsoSzPnouR3RvfrRnf2UYkEcrG4NfC9qt6lqn8FZohIq1AHFkl1L3yAddWTGbJxPNvXVsz+9cpk3svuJpqeV4ZuHXWaQ88r3Ghn+wo2JjOmhOa+AFVruwHoQ6njRVCvrTuAqUT9kwWSOj8EfO+yyoF2LtIAAB7xSURBVPVeq7xi46hxxRvsJ5HYD/8QXeOXHtzjbp5Jusz1yxRKZ9zpOrL70c4KTBnsTXdH6MlXBT7mQGnFxMKA22DbMtjwQ2jXFUaBJII4r4sIALznZbxdr/w7pUlLvur4CHUPb2X/hzdVquxfpCVvQ86h4DYZLUzdltD9ctfkb//20K/PVE4LXgE0PNssQNLlUOOUSnUAE0giyPBp7omIjAB2hS6k8uM3w0fyDKOpuW6Ku8O2ssvLdT+qlmfAKV3Cs84z/wp5OTArSq/HmLI5ctB1kd7xN+7AIhyqJLgBl9ZPh61Lw7POEAskEfwJ+LuIbBaRNOBvQAjaFJY/9arHE3fG7XyT24u8r+9zF6Qqs1++gszNbvyAcKnXBpJGwcLXICt8fQ6aSmL5e3Aos3RjDpRFynWu65VK0hldIK2G1qlqP1xX0p1V9XRgf8gjKyeuH9iWh6vcyi6pDx9eAwcic79bWMx7GWo1g9MuCu96B94FuYcrzY/KhElenmsyemp3aNE/vOuuVgdSrnVD31aC7lJK0s4qFhgpItOAxSGKp9ypUTWOq87pwXUHbyEvayd8ckPlHNA6Y427+NX7Ojc6UzjVbwvdRsKCV+FAVNQ6mmBY/x3s+qVsYw6URb+bQGJhzr/Dv+4gKzIRiEg1ERklIp8DK4AncfcUlHAA0Irtir4t2FOrMy8ljnEDXsx8PNIhBd/8CRBbFZKvjsz6z7zL3bo/+7nIrN9UPHNfdBdtu/w2Muuv1QS6j3I3RlbwA5hCE4GITAR+Ac4H/g20An5V1e9VtRIeEhcuoUosdwzuwPhd/dnaYjhMfwTWTY90WMFzaC8sneR6WAxWj40l1bADdL3E9RVTmavfTHBkrIHUae6mx7iAerwJjdNvd0O5zns5cjEEQVFnBF2BX3Edxv2sqrmcPJ5A1LgkuSltG9bgxl+vRBt2hI//WHluhFr6rutZMZwXif0ZeDccPehuDjKmKPNecmewwRpzoLQadnA3mc2fAIezIhtLGRSaCFS1O24AmlrANBGZCdQUkcbhCq48iYuN4a/nn8ZPGTl802W8q8b48FrX42FFlpfnjsKb9XFjOUdSo07QeYQ7uqqk/b6bIDi4x53BJo10I99F2oDbXculJW9HOpJSK/Iagar+7PUWehrwF+AtYL6IzA5LdOXM0K6N6da0NuPm5ZDzm2cgbS5MeyDSYZXNuu9gzzroW05aBJ91DxzZ7474jPFn8ZuhGXOgtJr3gRanw5znK+yBYcCthlR1odfXUEvgf0IXUvklItx9wWmk/5rNxAO9ofcNrsXAsveDeufx7qzDTF25nU27DwStzELNf9ldcOs0vPhlw+GULtBpmOv3PTuz+OVNdMkfc6DVmW6MgPLijDtgbxqs+DjSkZRKidsJqqoClaeTjRI6s30D+rWpx3PfpTLyzgdJ3LoEPh3jdqin3+Z2YiXssTPrcA7zN+xmVupuZqXu4uft7jaNGIEhXRszZmBbejQ/aSjnstu9DtZ+A2f9reyDfAfTwHtg9X9dFdGgv0U6GlOerP4v7NsCF5azlnvtzoOGnWDWM+4GyUg0Zy2DMDcYr/hEhHuGdOSSF2bz+rxt/PmaKe5i65x/w4dXQ93WbrDrHldAfKLfMg7n5LJkcyazU3cxa91ulqVlkpOnxMfFkNKyLndfcBrJLeoyY20G78zdxBc/badP63rcOLANZ5/WiJiYIG1kC151SSslwhfcCjo1CU67EOY+79pqJ9SKdESFUlXmb9jDewvSmLt+N2e2b8Co3i1IblEHKcvOIDMNVk92nall7XTDqfa80o0REc3mvuh+Yx0uiHQkJ4qJcdcKPvuTO7jqEKIxEUJEtIJ1ppaSkqILFy6MdBj88c2FzNuwm5n3nE2dxHjXT8/PU1yfOVsWQmJ9V3XU5wZyq9Vn1dZ9zFq3i1mpu1iwcQ+HjuYRI9CtWR0GtK3PgHYN6NWyLglVTjybyDqcw3vzN/PajxvYuvcQ7RvV4IaBbRjRowlV48owVsDhLHiysxt45nflcBzWrUtgwiDXKmTQvVCzfLVRyNh/mI8Xp/PBgjTW7zpAzapx9G1Tj9nrdnPwSC4dTqnBqN4tuKRnU+pWD/Bsa896WOXt/Ld692w27gbVG8L67131Y5tBkPwH11Ilks0mIyF9IbxyLgx5FPr9KdLRnCz3KDzTw/V5dO0XkY7mJCKySFVT/M4rLhF4w1ReiruP4NgZhKqOC2KMASsviWDN9v0MeWYGNw5sy71DOx6foYpums3B75+i+sZvOCJV+VTP4oXDQ9ikjWnfqAYD2jXg9Lb16dumPrWrVQlofUdz85iyfCsv/7Cen7fvp1HNqlw7oDW/79si4DJOsOBV+L874fpv3MWu8ujTP7mxjRFo0c+1KOo0DGo3i0g4uXnKzLUZvDc/jWmrd5CTp/RuVZdRvVtwYbfGJMbHkXU4hynLtvLegjSWpmUSHxvDBV0bc3nv5vRvU//ks7mMX2D1527nv/0n91qTZOg83F23qd/WvbY3HZZMdC1T9qa5A43uo11SaHhaeL+ISPnoelg7Fe5cBVVrRjoa/+Y8D1//Ha772m2z5UhZE8FXwF5gEW4sAgBU9YlgBhmo8pIIAP7y/lK+XLGNGXefTa4qs1J3Mzt1F7PX7Wb7vkO0lS3ckfgVQ/NmEKu5HG5/EQln/QWa9Sr1OlWVmWt3MWHGen5M3UWNqnFc3rs5153RmiZ1qgVaCLzQ310XGPND+a7P3Pmz20mu+hx2rnSvNU1xSaHz8LBUlWzJzOaDBWl8uDCNrXsPUa96PJcmN2VU7+a0a1T4Dmn1tn28vyCNT5dsYW/2UVrUS2RUSjNGtcqiwaav3GfKWO0Wbt7X7fg7DSu6F828XNfr5eK34OcvIO8oNO/nEkKXiyG+epA/fTmxdws8k+TG0B7ySKSjKdzhLHiqi2tOWq2e2z7rtnR/67Q8Pl27OcSW4gCuDMqaCFaoarm5PF+eEsHm3Qc554nvSYyPZd+hHMD1WNq/bX0GtHVH/S3rJyJZO1xzyAWvweG90HKAu7Dc/vwyDau3Yste/jNzPVOWb0OAYd2bMGZgGzqdWkyd+vof4K3hMOIFN0pYRbEr9fjR87Zl7rVTu3tnCiOgQbugrepobh7frt7BpPlpzFibAcAZ7Rpwee8WnNf5FOLjAv+/HTqSw9zZ0/l14Uck7fuBtjHbyEPY2zCFWsm/I7bLcNddQUllZbgzpsVvwe61ULWWu5aQ/IfI3xMSbNMedKOC3bak/F8n2bna9eT76yY3rGXmJnfNJ8+naanEuA4e67b0SRStjieK6g2DfoBW1kQwAXhOVX8KalSlVJ4SAcCEGeuYt34P/dvW5/S2DejYuGbhF3MP73c/2jkvwL50aHAanH6rGw2sDPW96b8e5NUfN/D+gjQOHsllYIeG3DiwDae3re//guV7V8DmOfCXVa5v9YpozwbXgmTV5+6aDECjLt6Zwgho1LHo9xdifUYW7y9I4+PF6ezKOkLjWglcltKMkSnNaV7P/8V/v1Rhy6LjZzOZm0BiyW7anxlVBvDk5vasOZDIKbWqMrJXcy5LaU6L+iUov+C6Ns917etXfuba2DdOcgmh20jXU2ZFdeSg25m+caE7gLp8YqQjKp28XNcTQaaXHPKTRH6iyNpx4vJVEk88g8h/3qQn1Dq1VCGUNRGsAtoBG4DDgOBakSaVKpoyKm+JoFRyj7rua2c9AztWQI3G7uJXr2vL9KPNPHiEifM28/qsjezKOkyXJrUYM7ANF3U7lbhY7wg2czM8090Nwj34H0H6QBG2N/14Utg8F1Bo0OF4Ujila5FHV4eO5vLlim1Mmp/G/A17iI0Rzu3YiMv7NOesDo2IDbSVVl4epM3zWvtMdsk+Js5d4O08wnXvXb0+4M44vvt5J+8vSOP7NTvJU3fGMap3c87vckrpGwJkZ7qhRhe/6a45xFVzVUbJf3BdNZe3asD8HWT+DtH3KPrXjT47SIFrv4SWYe5uOlyOHHS/zYKJIn/6iNd9xUVPuP6VSqGsicBvhaWqbipVNGVUKRJBPlV3Z+/sZ12rkPga0Osa12SyDBdEDx3N5bMlW5gwcz3rMw7QtE41rj+jNaN6N6f6jH+69d3xU8QuuobU/u3Hk8KmWW5M5Lqtj19TaJJ8bGe4aus+3luwmU+XbGH/oRxa1k9kVO/m/K5XMxolxrnO+A5luke29/fQ3uPPszOPL5Od6RLSwV0QGw9tz/V2/kOKHft5a2Y2Hy1K5/0FaWzJzKZuYhUuSW7G5b2b0/6UMlwU3brEnYEu/9DdrV2/vUsI3UeHr2sGVcj+9cSdWkmqTPKrSxp3dV2QRCNV163GrxuhdtNSt6ArUyLwCugOnOlNzlTVZaWKJAgqVSLwtW2Z64J5xSduR9X10jLX8+ap8suO/cxet5u0PQdJqBLLrXGfsrNuCt91j8i1/rBKOLKHFhnTabVjGk1+nU+M5rI/4VTWNzibFXtiyN63m7oxB2lfK5eW1Y9Siywke6/buR8pZuyl2HhIqOPO4BJqu+fVG0K7c921n1Lc+5CXp/yYuov3F6QxddV2juYqvVrWddckYkt/LSkuN5tWO6Zy2pZPOWXvUvIkjk0NB7GjTg/cCX7wiOaReDiDmtnp1MzeQs3sLcTnntgZ26EqddhfrQn7qzUjK6Ep+6vlP5pxIKExeTHhvYhakfRvW7/4a4CFKOsZwe3ADcAn3ku/BSaoakQ6jq+0iSBf5mZ3DWHxW65H0CDLVeGyI2NZpFHS5NBTmyzOi13E0Jj5nBmznHjJ5WhsNWIT6xJTrY7PTt3bsZ/wWu2T51epFtJqll1Zh/l08RYmLdjM+ozgbQftJJ1Rsd9zaewM6kloess8pFVI14Zs1kakeX/TtdGx6SxKeS3E8NDFXbmyX+nGZi5rIlgO9FfVA950dWCOXSMIsZzDrkvmIDuYG8PRmCj/IeYeola1eKQC3JClqq5FWrDv+8zLOV7vHGxVa7kqHhN0CfExpb5+VFQiCKSLCcHn/gHveTm74lQJxVUNyZ2jUZ4CPBWn6kFESnfDYLGqAAHed2IqvUASwevAPBH51Ju+GCiHfRIYY4wpjWITgao+KSLfA2fgzgSuVdUloQ7MGGNMeBSaCESklqruE5F6wEbvkT+vnqraEFLGGFMJFHVG8C7wG1wfQ76XqsSbbhPCuIwxxoRJoYlAVX/j/W0dvnCMMcaEW7FtvETk20BeM8YYUzEVdY0gAdfasIGI1OV4k9FaQCm6SjTGGFMeFXVGcCPu+kBH72/+43Pg+UAKF5EhIrJGRFJF5N4ilvudiKiI+L3ZwRhjTOgUdY3gGeAZEbm1NN1JiEgsLmGcB6QDC0RksqquKrBcTeA2YF5J12GMMabsArmP4DkR6Qp0BhJ8Xn+rmLf2AVJVdT2AiLwHjABWFVjun8B44K4SxG2MMSZIArlY/A/gOe9xNm6nPTyAspsCaT7T6d5rvmX3BJqr6pRiYhgjIgtFZGFGRkYAqzbGGBOoQHqG+h1wLrBdVa8FugOBdILjrz+iY/cjiEgM8BTw1+IKUtUJqpqiqikNG4apH3VjjIkSgSSCbFXNA3JEpBawk8BuJksHmvtMNwO2+kzXBLoC34vIRqAfMNkuGBtjTHgF0uncQhGpA/wH12ooC5gfwPsWAO1FpDWwBbgc+H3+TFXdCzTIn/b6M7pLVaOgj2ljjCk/ArlYfLP39CUR+QqoparLA3hfjojcAnwNxAKvqepKERkHLFTVyWUJ3BhjTHAUdUNZclHzVHVxcYWr6hfAFwVeG1vIsoOKK88YY0zwFXVGkD+obQKQAizDXQBOwrX5PyO0oRljjAmHQi8Wq+rZqno2sAlI9lrt9AJ6AqnhCtAYY0xoBdJqqKOq/pQ/oaorgB6hC8kYY0w4BdJqaLWIvAK8g7sP4EpgdUijMsYYEzaBJIJrgZuA273pGcCLIYvIGGNMWAXSfPQQ7g7gp0IfjjHGmHArqvnoB6p6mYj8xIlDVQKgqkkhjcwYY0xYFHVGkF8V9JtwBGKMMSYyihqPYJv3d1P4wjHGGBNuRVUN7cdPlRDupjJV1Vohi8oYY0zYFHVGUDOcgRhjjImMQJqPAiAijThxhLLNIYnIGGNMWAUyQtlwEVkLbAB+ADYCX4Y4LmOMMWESSBcT/8QNGvOLqrbGjVY2K6RRGWOMCZtAEsFRVd0NxIhIjKpOx/oaMsaYSiOQawSZIlID17XERBHZCeSENixjjDHhEsgZwQggG/gL8BWwDhgWyqCMMcaET1H3EfwbeFdVZ/u8/GboQzLGGBNORZ0RrAWeEJGNIvKoiNh1AWOMqYSKGqHsGVXtD5wF7AFeF5HVIjJWRDqELUJjjDEhVew1AlXdpKqPqmpP4PfAb7GBaYwxptII5IayKiIyTEQm4m4k+wW4NOSRGWOMCYuiLhafB4wGLgLmA+8BY1T1QJhiM8YYEwZF3Ufwd+Bd4C5V3ROmeIwxxoRZUb2Pnh3OQIwxxkRGIDeUGWOMqcQsERhjTJSzRGCMMVHOEoExxkQ5SwTGGBPlLBEYY0yUs0RgjDFRzhKBMcZEOUsExhgT5UKaCERkiIisEZFUEbnXz/w7RWSViCwXkW9FpGUo4zHGGHOykCUCEYkFngeGAp2B0SLSucBiS4AUVU0CPgLGhyoeY4wx/oXyjKAPkKqq61X1CK730hG+C6jqdFU96E3OBZqFMB5jjDF+hDIRNAXSfKbTvdcKcz1uvIOTiMgYEVkoIgszMjKCGKIxxphQJgLx85r6XVDkSiAFeMzffFWdoKopqprSsGHDIIZojDGmqPEIyiodaO4z3QzYWnAhERkM3AecpaqHQxiPMcYYP0J5RrAAaC8irUUkHrgcmOy7gIj0BF4GhqvqzhDGYowxphAhSwSqmgPcAnyNG+z+A1VdKSLjRGS4t9hjQA3gQxFZKiKTCynOGGNMiISyaghV/QL4osBrY32eDw7l+o0xxhTP7iw2xpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6KcJQJjjIlylgiMMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6KcJQJjjIlylgiMMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6KcJQJjjIlyIU0EIjJERNaISKqI3OtnflURed+bP09EWoUyHmOMMScLWSIQkVjgeWAo0BkYLSKdCyx2PfCrqrYDngIeDVU8xhhj/AvlGUEfIFVV16vqEeA9YESBZUYAb3rPPwLOFREJYUzGGGMKiAth2U2BNJ/pdKBvYcuoao6I7AXqA7t8FxKRMcAYbzJLRNaUMqYGBcsOEiu3YsUaqnIrUqwVrdyKFGt5LbdlYTNCmQj8HdlrKZZBVScAE8ockMhCVU0pazlWbnjKrGjlVqRYK1q5FSnWilhuKKuG0oHmPtPNgK2FLSMicUBtYE8IYzLGGFNAKBPBAqC9iLQWkXjgcmBygWUmA1d7z38HfKeqJ50RGGOMCZ2QVQ15df63AF8DscBrqrpSRMYBC1V1MvAq8LaIpOLOBC4PVTyeMlcvWblhLbOilVuRYq1o5VakWCtcuWIH4MYYE93szmJjjIlylgiMMSbKRUUiEJHXRGSniKwIcrnNRWS6iKwWkZUicnsQykwQkfkisswr88FgxOpTfqyILBGRKUEsc6OI/CQiS0VkYRDLrSMiH4nIz9533L+M5Z3mxZj/2CcidwQp1r94/68VIjJJRBKCVO7tXpkryxKrv9+AiNQTkW9EZK33t24QyhzpxZonIqVq5lhIuY9528FyEflUROoEqdx/emUuFZGpItIkGOX6zLtLRFREGgQh1gdEZIvP9nthSWMtlKpW+gcwEEgGVgS53FOBZO95TeAXoHMZyxSghve8CjAP6BfEmO8E3gWmBLHMjUCDEPzf3gT+6D2PB+oEsexYYDvQMghlNQU2ANW86Q+Aa4JQbldgBZCIa9gxDWhfyrJO+g0A44F7vef3Ao8GocxOwGnA90BKEGM9H4jznj9a0liLKLeWz/PbgJeCUa73enNcY5lNJf19FBLrA8BdZd2u/D2i4oxAVWcQgvsTVHWbqi72nu8HVuN2CmUpU1U1y5us4j2CckVfRJoBFwGvBKO8UBKRWrgfw6sAqnpEVTODuIpzgXWquilI5cUB1bz7YRI5+Z6Z0ugEzFXVg6qaA/wA/LY0BRXyG/Dt4uVN4OKylqmqq1W1tHf+F1XuVO87AJiLuy8pGOXu85msTil+a0XsX54C7glymSERFYkgHLyeU3vijuDLWlasiCwFdgLfqGqZy/Q8jdsw84JUXj4FporIIq87kGBoA2QAr3tVWa+ISPUglQ2uqfKkYBSkqluAx4HNwDZgr6pODULRK4CBIlJfRBKBCznxJs2yOkVVt4E7qAEaBbHsULoO+DJYhYnIwyKSBlwBjA1SmcOBLaq6LBjl+bjFq8p6raRVeUWxRBAEIlID+Bi4o8ARRqmoaq6q9sAd9fQRka5BiPE3wE5VXVTWsvwYoKrJuJ5m/ywiA4NQZhzu1PhFVe0JHMBVX5SZd4PjcODDIJVXF3d03RpoAlQXkSvLWq6qrsZVg3wDfAUsA3KKfFMlJyL34b6DicEqU1XvU9XmXpm3lLU8L2nfR5CSio8XgbZAD9wBxxPBKtgSQRmJSBVcEpioqp8Es2yvKuR7YEgQihsADBeRjbieYM8RkXeCUC6qutX7uxP4FNfzbFmlA+k+Z0Mf4RJDMAwFFqvqjiCVNxjYoKoZqnoU+AQ4PRgFq+qrqpqsqgNxVQVrg1GuZ4eInArg/d0ZxLKDTkSuBn4DXKFepXmQvQtcGoRy2uIOCpZ5v7dmwGIRaVyWQlV1h3eQmAf8h+D8zgBLBGUiIoKrw16tqk8GqcyG+S0iRKQabifzc1nLVdX/UdVmqtoKVy3ynaqW+ahVRKqLSM3857iLemVunaWq24E0ETnNe+lcYFVZy/WMJkjVQp7NQD8RSfS2iXNx14vKTEQaeX9bAJcQ3Lh9u3i5Gvg8iGUHlYgMAf4GDFfVg0Est73P5HCC81v7SVUbqWor7/eWjmtUsr0s5eYnbc9vCcLv7JhQXIEubw/cj2cbcBT3T7k+SOWegasfXw4s9R4XlrHMJGCJV+YKYGwIvo9BBKnVEK4uf5n3WAncF8Q4ewALve/iM6BuEMpMBHYDtYP8nT6I24msAN4Gqgap3Jm4BLgMOLcM5Zz0G8B1+f4t7izjW6BeEMr8rff8MLAD+DpIsabiuqzP/52VpnWPv3I/9v5ny4H/Ak2DUW6B+Rspeashf7G+DfzkxToZODVY2691MWGMMVHOqoaMMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMBWG19VCfs+L2wv0xBgfYBmv+9ybUNgyfxaRK4IU848issYnzveDUa5P+eml6YnTGF/WfNRUSCLyAJClqo8XeF1w23Ww+1MqFRH5EbhFVZeGqPx0oKsGt0M+E2XsjMBUeCLSzuuz/yVgMXCqiEwQkYVe3/hjfZb9UUR6iEiciGSKyL/Ejf0wx+cu3ofE6/vfW/5f4saIWCMip3uvVxeRj733TvLW1aMEMb8jIi+KyEwR+UVEhnqvVxORN8WN77A4v98mL96nvM+5XERu9inuDq9jvuUi0sFb/hwvtqVeOcHssM9UMpYITGXRGXhVVXuq6w30XlVNAboD54lIZz/vqQ38oKrdgTm4Xi39EVXtA9zN8Y7EbgW2e+/9F67n2cK871M19C+f15sDZwHDgAkiUhXXJ/4RVe0GXAW87VV73YTr0K67qibh+ovKt0Ndx3yv4MabwIt1jLrOCwcCh4qIz0Q5SwSmslinqgt8pkeLyGLcGUInXKIoKFtV87szXgS0KqTsT/wscwbezlhdV8Mri4htlKr28B6+Pah+oKp56vrvTwPae+W+7ZW7EjeuQTtcn1MvqWquN8+3r3p/8c0CnhaRW3GDr+QWEZ+JcpYITGVxIP+J15HY7cA53tHzV4C/oSOP+DzPxXV97c9hP8tImaJ1Cl6g0yLKFT/L5zspPlV9CLgRqAEsKNC5mjEnsERgKqNawH5gn9dj4wUhWMePwGUAItIN/2ccxRkpTgdcNdFaYAZugBREpBNuONRUYCpwk4jEevPqFVWwiLRV1eWq+r+4TgyLbCllolthR0DGVGSLcT12rgDW46pJgu054C0RWe6tbwWwt5Bl3xeRbO/5DlXNT0ypuB1/I1x9/hEReQ54WUR+wvU8+Qfv9ZdxVUfLRSQHN0jJS0XEd5eInIkbjW45LpEY45c1HzWmFMSNTRynqoe8apepuIHlAxpBTNygQB+p6mehjNOYQNgZgTGlUwP41ksIAtwYaBIwpryxMwJjjIlydrHYGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjotz/B8JzJ5H7lKq/AAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] + "text/plain": "" + }, + "metadata": {}, + "output_type": "execute_result", + "execution_count": 80 + }, + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaUAAAExCAYAAADcJb37AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2dd5icVdn/P/f2zWaTbHY3ddN7IUAIhNCr9KLSRAWkKQi8IqgIigq8P1EElapYaAq8IiIBqVKkJUAIGEgPqZueTc9m+/3745zZeXayu9lspjyze3+ua655ypl5vnNm5vmecp9zRFUxDMMwjDCQkWoBhmEYhhHBTMkwDMMIDWZKhmEYRmgwUzIMwzBCg5mSYRiGERrMlAzDMIzQkJVqAa1RUlKigwcPTrUMwzCMtOKjjz7aoKqlqdbRHkJtSoMHD2bGjBmplmEYhpFWiMiyVGtoL9Z8ZxiGYYSGuJmSiPxZRNaJyGctnBcRuVtEFonILBGZGK9rG4ZhGB2DeNaUHgZObOX8ScAI/7gceCCO1zYMwzA6AHEzJVV9C9jYSpIzgEfVMR3oISJ943V9wzAMI/1JZp9Sf2BFYL/cHzMMwzAMILmmJM0c22WKchG5XERmiMiM9evXJ0GWYRiGERaSaUrlwIDAfhmwKjaRqj6oqpNUdVJpaVqG2RuGYRjtJJnjlKYCV4nIk8BkYIuqrk7i9Q3DMFJCQ4NSVVfPzpp6Kmvq2Vkb3a6qrae6rp6q2ga/3fS5qrah8Xzwubq2gaoWntOZuJmSiDwBHAWUiEg58BMgG0BVfwe8AJwMLAIqgW/E69qGYRjxoK6+gR3V9WyvqWNHdR3bq92z265nR3UdO2ujZlJZUxfYdkazM2Y78twesjKEvOxMcrMyGp9zszPJy84gNyuDngU5jefysjLJzXbbN8U5X5JJ3ExJVb+ym/MKfDte1zMMwwBQVapqG9haVcvWnbX+uY6tVbWNphIxlO1VdY2GEzweMaDqurbXMvKzM+mSk0mef87PySQ/O5OeBTmUFUWPd8nJiqbJjqaLvCZiKHnZGbsYUFZm+3pYzJQMwzD2guq6+kYj2bIzYi51TUxmS+O2O7ctcK6mfvdmUpCTSUFuFl1zsyjIzaIgN5P+PfL8tjvetXE7s8nxgpzIOWcyuVkZZGQ0F7tl7C1mSoZhxJXa+gY2VdawaUctG3fUsKmyxj3vqGFTZW10P3B8R03rzVvZmUL3/Gy65WVTmJ9Nt7wsyoryG491y8/yz+5c9/xsCvOyG42kICfLTCRNMFMyDKNV6uobqNhRw7qt1azbVsX6bdVsrHRmsnHHriazraquxffqmptFUUE2PbvkUNQlh2GlXSnqkkNRl2x6dImYSlOT6Z6fTW5WBiJmKp0BMyXD6KTsqK5j/bZq1m2Lms26bdXRY1ur2LC9moodNeguIwpp7D8pKsimqEsOg4q7UNQlxx3rkk1RQY4znwJ3rEeXbHKzMpP/QY20wkzJMDoYNXUNrNlSRfnmSlZtrmLdtirWba1m/fZq1gdqO801mWVlCKWFufQqzKWsqAv7DyyiV2Fu47Fe3fIo6ZpDcUEu+TlmMEb8MVMyjDRCVdm6s67RcFZt3slK/1i1eScrN+1k/fbqXWo2XXOzGs1lfP/u9CrMo1e3XEq75rrnwlx6FebRIz/b+l6MlGKmZBghoq6+gbXbqlm5aecuhhMxndgaTk5WBv175NOvRx5Hjiylf1E+/Xrk+2P59O6WS5cc+6sb6YH9Ug0jydTWN1C+aSdLN+xg8YYdLN2wgyX+sWZrFfUNTas5RV2y6V+Uz+DiAg4ZVkJZjOkUF+RY7cboMJgpGUYCaGhQVm+tYsn6HSyp2MGS9TtYWuGMZ8XGSuoCxlOYl8XQkgImDS5iQFGXQE0nj3498q2WY3Qq7NduGO1EVVm/vZqlGypZsmE7S/zz0g2VLK3Y0WR2gPzsTAaXFDCmbyEn79OHwcUFDC0tYHBxAT0Lcizc2TA8ZkqG0QY2V9Ywd/U25q3Zyjz//Pn6HWyvjo7Jyc4UBvbswpCSrhwxsoQhJV0ZXNKFoSVd6d0t14zHMNqAmZJhBKirb2DJhh3MXbONuau3Mm/1Vuat2cbqLVWNaXoW5DC6TyFfntifISUFDC4pYGhJV/r1yGv3XGWGYTjMlIxOy8YdNcxdvdWZzxpX+1mwdjs1vtktK0MY3qsrBw8tZnSfQkb37caYvoWUdrVaj2EkCjMlo8NTW9/A5+u3M2/1Nuau2eqa4VZvZd226sY0JV1zGdO3kIsOGczoPoWM6duNYaVdycmymo9hJBMzJaPDsW5bFTOXbebj5ZuYuXwTs8q3NAYd5GRmMLxXVw4bUcKYPt0Y07cbo/oUUlqYm2LVhmGAmZKR5tTWNzBv9TZmegOauXwTKzbuBFzgwbh+3fnq5EFMKOvOmL7dGFpaQLb1+xhGaDFTMtKKiu3VzFy+mZnLN/HRsk3MKt9MlV/+uVdhLgcMKuKCgwczcVAPxvXrTl62zc9mGOmEmZIRWurqG5i/dhszl21qNKJlFZWAC0IY168b5x04kImDipg4sAf9e+RbAIJhpDlmSkZo2FlTzwdLN/LBkgpmLtvMf8s3U+nneSvpmsvEgT34ykEDOWBQEfv0t1qQYXREzJSMlKGqzF+7jbcWrOetBRv4YOlGauoayMwQxvQt5OwDynwtqIiyIqsFGUZnwEzJSCoV26t5Z9EG3lqwgbcXrm8Myx7RqytfP3gQR4ws5cDBRTbfm2F0UuyfbySUmroGZi7fxFsL1vP2wg18tmoLqtCjSzaHDS/hiBGlHD6yhL7d81Mt1TCMEGCmZMQVVWVpRaU3ofVM+7yCHTX1ZGYIEwf24LvHjeTwkaXs0787mbbcgmEYMZgpGXvN1qpa3ltUwVsLnRFFxgkN7NmFL07sz+EjSjlkWDGFedkpVmoYRtgxUzLaxYqNlTw3axWvz13Hxys2U9+gFORkMmVYCZcfPpQjRpYyqLgg1TINw0gzzJSMNrNq807+NWs1z89axX/LtwCwT//uXHHkMA4fUcLEQUU2W4JhGHuFmZLRKuu2VvGvT1fz/KzVfLRsE+CM6IcnjeaUCX0pK+qSYoWGYXQkzJSMXajYXs2Ln63h+VmreH/JRlRhdJ9CvnfCKE7Zpy+DS6xZzjCMxGCmZABuZdWXZ6/h+Vmree/zCuoblGGlBVxzzAhO27cvw3sVplqiYRidADOlTszWqlpenb2W52et4p1FG6itVwb27MK3jhzKqRP6MbpPoc2iYBhGUjFT6mRU1tTx77nreP6/q3hzwXpq6hro3yOfbxw6hFMn9GWf/t3NiAzDSBlmSp2A+gblPwvW8fRHK3lt3lqqahvo3S2Xr04eyKkT+rH/gB5k2EBWwzBCQNxMSUROBH4LZAJ/VNXbY84PBB4Bevg0N6jqC/G6vrErmytr+NuMFTw2fRkrNu6kuCCHsw8YwKkT+nLg4J5mRIZhhI64mJKIZAL3AccD5cCHIjJVVecEkv0I+JuqPiAiY4EXgMHxuL7RlM9WbuHRaUt59pNVVNc1cNCQntxw4hi+MK63jSMyDCPUxKumdBCwSFUXA4jIk8AZQNCUFOjmt7sDq+J0bQOorqvnxU/X8Oi0pcxcvpn87Ey+fEAZF0wZxOg+3Xb7esMwjDAQL1PqD6wI7JcDk2PS/BR4RUSuBgqA45p7IxG5HLgcYODAgXGS13FZtXknj7+/nCc/XM6G7TUMKSng5lPH8uUDyuieb3PNGYaRXsTLlJrrnNCY/a8AD6vqnSIyBXhMRMarakOTF6k+CDwIMGnSpNj3MHAzcU9bXMGj7y3j1blraVDl2NG9uGDKYA4bXmJ9RYZhpC3xMqVyYEBgv4xdm+cuAU4EUNVpIpIHlADr4qShw7O9uo5nZpbz6LRlLFy3nR5dsrn08CF8bfIgBvS06X4Mw0h/4mVKHwIjRGQIsBI4Dzg/Js1y4FjgYREZA+QB6+N0/Q7NonXbeWzaUp6euZLt1XXs0787d5w1gdP27Udedmaq5RmGYcSNuJiSqtaJyFXAy7hw7z+r6mwRuQWYoapTgeuAP4jItbimvYtU1ZrnWqCuvoHX5q3j0WlLeXdRBTmZGZwyoS8XTBnEfgN62ABXwzA6JBJmX5g0aZLOmDEj1TKSSlVtPY+/v5w/vbOElZt30q97Hl89eBDnHjiAkq65qZZnGEYaICIfqeqkVOtoDzajQ0iorW/gqRnl3PP6QlZvqWLykJ78+NSxHDemF1k2tsgwjE6CmVKKqW9Qnv1kJb/590KWb6xk4sAe3HnOvhwyrCTV0gzDMJKOmVKKUFVe+mwNd726gIXrtjO2bzf+fNEkjh7Vy/qLDMPotJgpJRlV5c0F67nzlfl8tnIrw0oLuO/8iZw0vo+NLzIMo9NjppREpn1ewZ2vzGfGsk0M6JnPnWfvy5n79yfTzMgwDAMwU0oKHy/fxJ2vLOCdRRvo3S2X284czzmTBpCTZQEMhmEYQcyUEsjc1Vu585X5/HvuOooLcvjRKWP42sGDbMCrYRhGC5gpJYDP12/n168u4PlZqynMy+L6L4zkG4cOoSDXstswDKM17C4ZR1ZsrOTu1xby9Mxy8rIzuero4Vx2+FC6d7HZug3DMNqCmVIcWLu1intfX8STHy5HRPjGoUO44qhhNgODYRjGHmKmtBdU19XzwJuf88Cbn1PfoJxz4ACuPmY4fbvnp1qaYRhGWmKm1E4+WraJG56excJ12zl1Ql++f8JoBhbb8hGGYRh7g5nSHrK9uo47XprHo9OX0bdbHg9ddCBHj+6ValmGYRgdAjOlPeD1eWu56ZnPWLO1igunDOb6E0bR1SLqDMMw4obdUdvAhu3V/Oy5OTz331WM7N2V+756CBMHFqValmEYRofDTKkVVJWnZ67ktn/NobK6nu8eP5JvHTnMZmIwDMNIEGZKLbC8opIbn/mUdxZtYNKgIm7/8j4M71WYalmGYRgdGjOlGOrqG3jo3aXc+ep8sjIyuPXM8Xz1oIE2g7dhGEYSMFMKMHvVFm54+lM+XbmF48b04tYzx9uYI8MwjCRipgRU1dbz29cW8uBbiynqks1950/k5H362GJ7hmEYSabTm9J7n2/gxn98ytKKSs6ZVMaNJ4+hR5ecVMsyDMPolHRaU9pSWcvPX5zLkx+uYFBxF/566WQOHV6SalmGYRidmk5nSqrKS5+t4eaps9m4o4ZvHjmU7xw7kvwcW+PIMAwj1XQqU1q/rZqbnvmUV+asZVy/bjx00YGM79891bIMwzAMT6cxpY+Xb+KKv8xkU2UNPzxpNJccNoSsTBsEaxiGESY6hSk9+cFybn52Nr275/LMlYcytl+3VEsyDMMwmqFDm1J1XT0/nTqHJz5YzuEjSrjnK/tbZJ1hGEaI6bCmtHZrFd/6y0d8vHwzVx41jOu+MIpMm5XBMAwj1HRIU/pw6Uau+MtMKmvqeOCrEzlpn76plmQYhmG0gQ5lSqrKY9OXcctzcxjQswuPXzaZkb1tElXDMIx0ocOYUlVtPTc98xlPzyzn2NG9uOvc/eien51qWYZhGMYeELeYaBE5UUTmi8giEbmhhTTniMgcEZktIo/H69orN+/k7N9N4+mZ5fzPsSP4wwWTzJAMwzDSkLjUlEQkE7gPOB4oBz4UkamqOieQZgTwQ+BQVd0kIr3ice33Pt/AVY9/TG1dA3+4YBLHj+0dj7c1DCOJ1NbWUl5eTlVVVaqlpBV5eXmUlZWRnd1xCuHxar47CFikqosBRORJ4AxgTiDNZcB9qroJQFXX7c0FVZU/vbOEn784jyElBfz+6wcwrLTr3rylYRgpory8nMLCQgYPHmyz87cRVaWiooLy8nKGDBmSajlxI17Nd/2BFYH9cn8syEhgpIi8KyLTReTE9l6ssqaOa578hNv+NZfjx/Tmn98+1AzJMNKYqqoqiouLzZD2ABGhuLi4w9Uu41VTau6XpM1cawRwFFAGvC0i41V1c5M3ErkcuBxg4MCBu7zp8opKLn9sBvPXbuN7J4ziyqOG2Q/ZMDoA9j/eczpinsWrplQODAjslwGrmknzrKrWquoSYD7OpJqgqg+q6iRVnVRaWtrk3H8WrOe0e99h1eadPHTRgXz76OEd8ksxDCN9UFWuueYahg8fzoQJE5g5c2az6W666SYGDBhA167WqtMa8TKlD4ERIjJERHKA84CpMWn+CRwNICIluOa8xW15c1XlvjcWcdFDH9C3ex7PXX0YR42KS5yEYRjGXvHiiy+ycOFCFi5cyIMPPsgVV1zRbLrTTjuNDz74IMnq0o+4mJKq1gFXAS8Dc4G/qepsEblFRE73yV4GKkRkDvAG8D1Vrdjde2+vruOKv8zkjpfnc+qEfvzjykMYVFwQD9mGYRiNLF26lNGjR3PhhRcyYcIEzjrrLCorK3f7umeffZYLLrgAEeHggw9m8+bNrF69epd0Bx98MH372uwyuyNug2dV9QXghZhjNwe2Ffiuf7SJ6roGzrzvXRav386PThnDJYcNseY6w+jg/Oy52cxZtTWu7zm2Xzd+ctq43aabP38+f/rTnzj00EO5+OKLuf/++1m5ciVvvPHGLmnPO+88brjhBlauXMmAAdHei7KyMlauXGkG1E5CPaPDonXbyd5Rw18umcwhtlS5YRgJZsCAARx66KEAfO1rX+Puu+/mn//8Z6uvceXtpljhuf2E2pTyszOZetWhlBV1SbUUwzCSRFtqNIki1kxEhGuvvbbVmlJZWRkrVkRHxJSXl9OvX7+Ea+2ohNqUhpYWmCEZhpE0li9fzrRp05gyZQpPPPEEhx12GNddd12rrzn99NO59957Oe+883j//ffp3r27Nd3tBbYeuGEYhmfMmDE88sgjTJgwgY0bN7YYSRfk5JNPZujQoQwfPpzLLruM+++/v/Hcfvvt17j9/e9/n7KyMiorKykrK+OnP/1pIj5C2iPNtYeGhUmTJumMGTNSLcMwjAQzd+5cxowZk1INS5cu5dRTT+Wzzz5LqY49pbm8E5GPVHVSiiTtFVZTMgzDMEKDmZJhGAYwePDgtKsldUTMlAzDMIzQYKZkGIZhhAYzJcMwDCM0mCkZhmEYocFMyTAMYy9o69IVH330Efvssw/Dhw/nmmuuaZye6KmnnmLcuHFkZGRgQ2DMlAzDMPaKti5dccUVV/Dggw82pn3ppZcAGD9+PP/4xz844ogjkik7tJgpGYZhkNilK1avXs3WrVuZMmUKIsIFF1zQONHrmDFjGDVqVEI+UzoS6rnvDMPohLx4A6z5NL7v2WcfOOn23SZL1NIVK1eupKysbJc0xq6YKRmGYXgStXSFLW/RdsyUDMMIF22o0SSKRC1dUVZWRnl5eatpDIeZkmEYhidRS1f07duXwsJCpk+fzuTJk3n00Ue5+uqrE/lR0hYLdDAMw/AkcumKBx54gEsvvZThw4czbNgwTjrpJACeeeYZysrKmDZtGqeccgonnHBC/D9YGmFLVxiGkXJs6Yr2Y0tXGIZhGEaCMFMyDMPAlq4IC2ZKhmEYRmgwUzIMwzBCg5mSYRiGERrMlAzDMIzQYKZkGIaxF+zt0hUbN27k+OOPZ8SIERx//PFs2rQJgHnz5jFlyhRyc3P51a9+lbTPk2rMlAzDMPaCvV264vbbb+fYY49l4cKFHHvssdx+u5tmqWfPntx9991cf/31SfssYcBMyTAMg9QtXfHss89y4YUXAnDhhRc2Hu/VqxcHHngg2dnZcf6k4cbmvjMMI1T84oNfMG/jvLi+5+ieo/nBQT/YbbpULF2xdu3axrR9+/Zl3bp17f6cHQEzJcMwDI8tXZF64mZKInIi8FsgE/ijqjY7/7yInAU8BRyoqjaxnWEYTWhLjSZRpGLpit69e7N69Wr69u3L6tWr6dWrVzw/UtoRlz4lEckE7gNOAsYCXxGRsc2kKwSuAd6Px3UNwzDiSWTpCqBx6Ypf//rXfPLJJ7s8brjhBsAtXfHoo4+iqkyfPn23S1eoKo8++ihnnHFG4+sfeeQRAB555JHG452VeAU6HAQsUtXFqloDPAk0l7O3Ar8EquJ0XcMwjLiRiqUrbrjhBl599VVGjBjBq6++2mh2a9asoaysjLvuuovbbruNsrIytm7dGudPHD7i1XzXH1gR2C8HJgcTiMj+wABVfV5EOleMo2EYaUFGRga/+93v9ug1IsJ9993X7LlPPvmkcXvSpEnNTvhaXFzMa6+9tsvxPn36NGny6yzEq6bUXI9dY8+eiGQAvwZaX8LRpb1cRGaIyIz169fHSZ5hGIaRDsTLlMqBAYH9MmBVYL8QGA+8KSJLgYOBqSKyyyJUqvqgqk5S1UmlpaVxkmcYhtE6tnRFOIiXKX0IjBCRISKSA5wHTI2cVNUtqlqiqoNVdTAwHTjdou8MwzCMIHExJVWtA64CXgbmAn9T1dkicouInB6PaxiG0bFpbiyP0TodMc/iNk5JVV8AXog5dnMLaY+K13UNw0h/8vLyqKiooLi42AaVthFVpaKigry8vFRLiSs2o4NhGCknMrjUgpv2jLy8vCbTF3UEzJQMw0g52dnZDBkyJNUyjBBgs4QbhmEYocFMyTAMwwgNZkqGYRhGaDBTMgzDMEKDmZJhGIYRGsyUDMMwjNBgIeGGYRjpTM0O2LQMNi2Fzf45jTFTMgzDCDMN9bB1pTObTUt3NaAdMQOOc7omX2McMVMyDMNIJaqwc1PUdCJmEzGgLSugoS6aXjKhexkUDYZRJ7nnosHQwz936Qk3pW/PjJmSYRhGoqmvdeaycUnAcJZEjac6ZkXZLsXOYPrtD+O+6I1nkHvuVgaZHffW3XE/mWEYRjKJ1HaaM54t5aAN0bSZuVGTGXAw9BwCPQZFzSe3MBWfIBSYKRmGYbSF+jrYWh41nFjzqdrSNH1BqTedyTDhXCgaEm1qK+wLGenbxJZIzJQMwzAi1FRGTWbjkqbPm5c37dvJyI7WdsomecMZYrWdvcRMyTCMzoMqVFbEGM7S6Pb2NU3T53V3RtN3Xxh7pmtmixhPt36QkZmKT9GhMVMyDKNjEQmh3rgENi5uWttpLqigsJ8zm+HHQc/BznQi5tOlZ0o+QmfGTMkwjPSjvtY1p21cHDWfiAFtWgr1NdG0mTnRIIKBU5qaTtEgyM5P1acwmsFMyTCMcFK70wcUxBjPxsU+mq0+mja7AHoOhdLRbuxOz6HefIZaM1uaYaZkGEbqqN7W1Gw2Loka0LZVTdPm9XAmU3agi2br6U2n51AX6SaSms9gxBUzJcMwEkvV1oDpfB41nYrPYce6pmkLejmTGXqUNxzr3+lsmCkZhrH3VG31hrMYKoIGtHjXudkK+zrDGXlCtKYTeeSm97xtxt5jpmQYRtuIGE/F59GaT2S7ckPTtIX9nMlE+nd6DovWfHIKUqPfSAvMlAzDiFKzI2A23oAi27E1nm79ndGMPsU9Fw+LBhjkdEmNfiPtMVMyjM5GbZULnW7OeLatbpq2ax9nNiNPdM/Fw32tZ4iFUhsJwUzJMDoikXE8FYv8I2BAW8oBjabtUuIMZ+jRUOyb2oqHWx+PkRLMlAwjXWlocDWboPFEtjcvazpPW153ZzQDp/hmtmFRA8rvkbrPYBgxmCkZRtip3NjUcII1n9rKaLqsfGc4fcbDuDOjTW3Fw104tY3jMdIAMyXDCAM1lb55LWI6i6PbOzdG00mmmy6neDgMOcL383jjKexnyyEYaY+ZkmEki4Z6159TsRA2RMzHb28tb5q2sJ8zm7FnOMOJPIoGQWZ2avQbRhIwUzKMeNPY3LYQNixs2uRWXx1Nl9vNGc2gQ6BkhK/xjLAAA6NTY6ZkGO2hrtpNlxOs7US2Kyui6TKyfHPbCBh2jDefEc6Muvayfh7DiCFupiQiJwK/BTKBP6rq7THnvwtcCtQB64GLVXVZvK5vGHFHFXZs8KazwNV6Nix0+5uWgjZE0xb0coYz+lRnOBHzseY2w9gj4mJKIpIJ3AccD5QDH4rIVFWdE0j2MTBJVStF5Argl8C58bi+YewVdTVuMOkGbz4Vi6ImVLU5mi4rz0Wz9ZkA47/sTKfE9/XkdU+dfsPoQMSrpnQQsEhVFwOIyJPAGUCjKanqG4H004GvxenahtE2dlR404nUfLz5bFradG2ern1cTWf8l7zxjHTm032ArctjGAkmXqbUH1gR2C8HJreS/hLgxThd2zCiNNQ7k4nUehofC5uGVmfmBsb0fDFqPMUjIK9byuQbRmcnXqbUXG+tNnMMEfkaMAk4soXzlwOXAwwcODBO8owOR82OaB/PhgWwYX400i24FHZBL2c4Y0/3xjPSNbf1GGi1HsMIIfEypXJgQGC/DFgVm0hEjgNuAo5U1erY8wCq+iDwIMCkSZOaNTajk6AK29fF1Hh8rWdLoGIuGW5m6pKRMOL4puZjC8MZRloRL1P6EBghIkOAlcB5wPnBBCKyP/B74ERVXbfrWxidlkiT2/r50RpPxICqtkTTZRe4vp7IuJ6I+fQcClm5KZNvGEb8iIspqWqdiFwFvIwLCf+zqs4WkVuAGao6FbgD6Ao8JW5sxnJVPT0e1zfShNoqH9k2H9YviD5XLGo6qLRrb2c2+5ztjWcElIyCbv1sXI9hdHDiNk5JVV8AXog5dnNg+7h4XcsIOVVbXG1n/XxYP8/VeNbPdzNXN47tETeotHQUDD/WPZeMcgZks1YbRqfFZnQw2oeqW4l0/Tzf7LYg+hxcKC4zx/Xt9N0XJpzjaj6lo9wxWyTOMIwYzJSM1lGFrat8U9v8qAmtnwc7N0XT5RS6Ws7Qo6F0pKv1lI6CHoMg035mhmG0DbtbGI6GBhfRFms86+dDzbZouvwiKB0DY8+E0tFRA7L+HsMw4oCZUmejMdJtnn8siPb7BBeMiwQb7Hueq/GUjnaPghIzH8MwEoaZUkelvs7N57Z+HqybB+vn+j6fhU0j3br1d6ZzwEVR8ykZaeN7DMNICWZK6U5DvVtCYf1cbz7+EWs+PQa6ZrZhR0drPSUjbUodwzBChZlSuhBpdls3N2BAPtotaD7dB0Kv0d58xrjtklG2aJxhGGmBmVLYCPb5rJsbbX7bxVa1locAACAASURBVHwGuNrOsKOc+USCDnILU6XcMAxjrzFTShWqsKU8UPOZC+vmuMCDup3RdN0HuL6eoUdCrzHegMx8DMPomJgpJZrIpKLr5vhazxxvQPOahloX9nW1nUkXO/PpNcb6fAzD6HSYKcWTyo0xNR//CK7jk98Teo9zoda9xkCvsa7fJ78odboNwzBCgplSe6jZ4Wo9a+dEm93WzYXta6Jpcrs50xlzmjceX/spKLVxPoZhGC1gptQa9bVQ8Tmsm+1MZ+0cZ0CbltK4hmFWfmBS0dFRA7IZDgzDMPYYMyXwQQcroqYTqflsWBBdxVQyoxOL7nd+tOmtaLCtYGoYhhEnOp8p7agIGM+caBNcMOig+wBnOsOPc8bTeywUj4DsvNTpNgzD6AR0XFOqrXIzW6+dA2s/8wY0G7avjabJL4JePuig99ho01te99TpNgzD6MSkvympwubl3nQ+8yY0261mqvUuTWaun+Xg2Kj59B7nJh21fh/DMIzQkF6mtHNztMazdna0+S3Y9NZjkDOcsad78xkPPYfamj6GYRhpQLjv1Ds3wb9/Gq39bC2Pnsvr7gxn3/OcCfUe55rebKYDwzCMtCXcprRpKbx3r5vZYNAUbzzegCzk2jAMo8MRblMqHQ03fgxZOalWYhiGYSSBjFQLaJXsfDMkwzCMTkS4TckwDMPoVJgpGYZhGKHBTMkwDMMIDWZKhmEYRmgId/SdYRhGGqGq1DXUUdtQS53WUd9QT11DXfSh0e16rY+mjTkWOR7cb+49mnuf+shMNmmKmZJhGElHVanX+l1vwg31u9x0d7kJ+zSt3sybe59m3qvF4wFDiRhMk+u38JoGbUhqPmZlZJGdkU2mZJKVkdX4SGfSW71hdAJUdddSt8bcgGNukm0tZbd6vBVzqG+op1Zr23SjbslMkkmWZJGZEbhxS1aTm3hkvzGNuJt9fla+OxY4n52R3eprgvtN0mdkNTGP5sxkF12B10TeJ7KfKZlICxMICOk7sYCZktGh2N0NPHhzDN7AY0vbrR3f0xt4a2awuyacyGuTSeMNsI038qyMLPKy8twNWAI34MDNvMWbcDPvG3sTbvV4QGPQLGJ1t3TzNsKHmVInp6UbdGs36dibfrDEHHvT3+Mbvn+/1triW6wFpPgGHltqbulmnJ+V32rJuKVjzZXSg9cMGkJbbu57Wvo2jGQQN1MSkROB3wKZwB9V9faY87nAo8ABQAVwrqoujdf1k0GwE7Olm+nuSsGxbdZNzrWxdN2WtLu7cUeafTSyrHsSaEsJPPYmmZeVt8fNHa2V6BubYGJu4JFrxt7AW7q5Z0qm3cANIwHExZREJBO4DzgeKAc+FJGpqjonkOwSYJOqDheR84BfAOe29r7barbxytJXmi0p73FzSgvGEVtab80kkhnVkiEZzd5wW2uiyMnIoUtWl11Kw601ecSW9HdXym7LTb+lJhm7gRuGsTviVVM6CFikqosBRORJ4AwgaEpnAD/1238H7hURUdUWi+rLty3nuv9c1yYBu2s+ae5mGenEbK7UHEzbluaPNt+sI6+T7Kam4tNHOkozxIaQGYbR+YiXKfUHVgT2y4HJLaVR1ToR2QIUAxtaetNh3Yfx9OlPN9teHnszt1K4YRhG+hMvU2rOEWJrQG1Jg4hcDlwOMHDgQEYWjdx7dYZhGEZaEK82onJgQGC/DFjVUhoRyQK6Axtj30hVH1TVSao6qbS0NE7yDMMwjHQgXqb0ITBCRIaISA5wHjA1Js1U4EK/fRbwemv9SYZhGEbnIy7Nd76P6CrgZVxI+J9VdbaI3ALMUNWpwJ+Ax0RkEa6GdF48rm0YhmF0HOI2TklVXwBeiDl2c2C7Cjg7XtczDMMwOh4Wd2wYhmGEBjMlwzAMIzSYKRmGYRihQcIcACci24D5qdbRBkpoZRBwiDCd8cV0xpd00JkOGgFGqWphqkW0h7DPEj5fVSelWsTuEJEZpjN+mM74YjrjRzpoBKcz1RraizXfGYZhGKHBTMkwDMMIDWE3pQdTLaCNmM74Yjrji+mMH+mgEdJH5y6EOtDBMAzD6FyEvaZkGIZhdCLMlAzDMIzQsFemJBLe5VHDrC1C2DWGXR+kh0YwnfEmzDrDrC0daFfmicgwEempqg1+PzO+stpPmLVFCLvGsOuD9NAIpjPehFlnmLWlE3sc6OBLAdOAncDDqvpwAnS1izBrixB2jWHXB+mhEUxnvAmzzjBrSzfaU1P6Om4F2Z8AU0TkDREpEk985XUobRHCrjHs+iA9NILpjDdh1hlmbemFqrb5AfQE3gO+6PfHA/8AugbSZO7Je8brEWZt6aIx7PrSRaPp7Fw6w6wtHR97WlO6Glilqs+ISBZwGLAQOFVEbhSRIlWth5S0p4ZZW7poDLu+dNFoOjuXzjBrSzvabEoiUopbwvzn/tDBuOrqMlV9EqgE3hSRawFUtT5Z1db2aPOvE/+c8B9K2DW2V1/g9X0SqS9dNHYynSWdXefe/q+NXWmzKanqeuBwVf1IRHoARwMKbBeR3wHvAEcCI0TkXRE5TX29VUTKROR8EXlMRE6L94dojzb/ukiUx0ki8kEitKWLxvbqAxCRgcA3RGS6iJySCH3porET6cwBzhSRGSJycmfVuTf/awmEjnudBrS9TwnICGyXAjcCZ/n904C3gd/h2lcnAH8DvogzvquBp4HLgA+AW4DseLVBtldb4DWDgd8DHwFjE9FOGnaNe6MP6ALkAqcAM4CLwpaHydLYCXR+KeY9TgfeB77dGXW2Q9tTkfOB113stf0ByE1UPqbLY2+/kOyY/f8BPgG+CwiQ549fD9zktwcDrwBdEvrBWtHm93MD5wYA3wFui3lNQjsn91Zj8A+RIn05sXkFHAM8nOi8i5PG4A1FQqwzaZ3kbf1NAl0jmoF9cUaQ5f/3WemgMxXafP5EhuJc7o/9C3gI+BFmSu0ePCsAqlrr9zP9/m9xVdXewBSgwDdBHAp8S0RuBH4NrFPVyvZcOw7aikXkRFWt9uczcO3A44BnAu8zFnhBRL4VVo3+XL6IHJIifTWR12i0Hf96oFJdn2KJiBwkIhfEU1+cNG5T1YZIE4r6u0QIdVb5vBzqm8AvSbHOav+SAcAbInIrcA+wVVXrgCOA1xPxv4mTzp1eZ76IFIvIF5KorRdwIDBRRG4HxuBqTq8Cy4FXVLVaRPqJyMmJ+q7DTlxnCReRzMAfChH5GZCJKwmsBR7D/TD+qapVPk2WqtaJyKnAelV9P26CWtEWOL4PbozBGlW9K3B8MC608zKgBrhCVRO6DHI7NN6GW555JFAMXKmq7yZDn4hI5EYuIqOA44CDcDXhY/yN9E1gNrAfUAFcpqprE6WvjRoPBIYBRwXSlQAnAl8Bvq+qsxOpcQ90DgWO9nlZDjyJu5FlA1ep6oJk6gwcE1VVEZkC3A78UlX/5c/1A8biavUVuOay7SHR+QtVfcGfewzYisvPHl7ntGRoE5HvAdcA3wTeBW4G5qrqH8UFuTyLa87bHygHvqmqWxOhLYzEdTl0jUbcCc6MuuKWNJ8GICK1uBJqVaBUUedf/hTwNdyXEXcC2jK9CUZ+KPvjfpT3eI2ijqXAUuB5EXkNV8pJqCm1VaPXeQmu3+EwVd0hIl8BjhCRz1R1SwL1ZaibRqW/iJwETAYmAm/impo+8Ol+iCuVftvrfQWXhwk1pTZofASY6dMNxNU+L8X9+Y8EuidS3x7ofBiY5dONxoUcXw8gIn8GRgEJN6WgThEZABSp6ix/bpqINABrAulXAauAf4vIO7hCU8JNqQ06FVjtay9XAoNV9XAAEfk6ME7cUue1CdLWWPBQ1TtEZBHwS6ABN8bpcREp9tpmqOo1XtvbuDw0U2ovPuMVaPA/hIOAB8VFviwFVvqkkabDel+Vna6qTwdKN41fYpy1RUzwJRGp8Zr+rqorAj/q7EAV/HhgPa50mnB2p9FrKgF+jCtlfVFE/k9VnxCRHokypIC+Br85HBewshQ4QV0UEl7fJOBwXKcvIjIMWIErqCSc3WkUkVwR2R/X1v8hcCeu2elxVX0vGRrbotNrzQaOB/JE5G5cf2wW0C8FOnsAD4jIa6p6s4hcDGwDPo9oDfxvzge2AJtDoPMSr3MhroB3FTBLRL4JPKSqj4lI10QYUkBbpCac4XefEZGtuFDyKbhWmYG47/WXPu3BwGpgU6J0hZFEz2b7/4AcEZmJu0G9DcwDV3rwJYhSXOngfP+a/f35hLTxB7gQV/O5FBjtr9ngn2tFZICIXISrnbyB+3FEflSISDcRmZBsjZ5bcaXPe3EhqH/0zaCbvbbI2KaEaVTVN3HRRB/iapMXBk5/Hdcc8bHPr15AIW5esMa2dnF9TlMSoW83GgcDvwD6qOp9uNrbV4neDDICeZhQja3pFJECr2kkLpKrP/B94Algqk+TlUSdnwJfAvqKyHRcnv0x8rsDMkVkrIj8ANdc9gcg0kwf+d+kQuf5wB98M+IluP/yD3A10zv8a7YnQ6eqNvhCdzdcM+3fvdYFwCRgtqouEBciPsRrzfeaIv+bfiJyXLy1hQZNXARKMLJpBDDCb/cD7gIK/f7vcKWY84DXcAZwUuQ9SECUTIy2I3E/zEygL3AOLhLmNeA3wOktvMdXgT8DByQh/47E3ZzygW44cy/z5/p6HaXJ1Bij7zDgB367J65EOiDwff8S+H/NvMeNuGWbByYhDw/D9RdF9g/ANZu84R/Xx74mGRpb0HmD3x6Ha1noHjh3n9+WFOssAwr89hhcFNlU//gJMLmF90iFzm5+uxj4NPJfAQ7BGWdeivJzEHCQ3+6La00o8fuTgF/h+rJjX/d74AEgP1HaUvmIe/NdBHXNYJF+o4Xgmk2Au4EKVd0mLmrsQmAxrkR1Da4Pqs433zVOAa/NBADESdt/gP/469yDK7X8UFW/EXxNpK/Mv3YIrmT7CTA3mRqBnSLyLlCE6wfphQskiJTwMpKhMUbfO7hBguBqQ+/gmmXzgbNwzRLXeg05qlrja3CjcSXFtfHW15pGn0cfAYf47/xSYJZ/WaaIRF6bcI2t6cT1zc4i2p9QhCv991DVzREtKdJZ7q9TiCu8HQ9cqKqPBV/jax6Sap2e7cBbuH4ccPlZjLu5V6UgP5cBy/xuAzAd2CYikcJxAy6EPfi/ORQXAHMdLgArIdpSScJMCZptghsKLFDVG/3+s7gIlGJcfP5saDSAZ0RklqreHMnwON9UI228wb6rc3A3z/NEpF5VfxWTPpLuJJyJvqw+tD0RP4oWNILrWP6riHyIqzn9XX1Um0bb1ROusTl9qrpTRBbjpvFfgLup3qmqq/35SOjzBbiBwG+oD99NVh76m1YmbqDqFHznsj8X7FdIisZWdH7mv+NZIrIA931OVd9cFtCSdJ2B/W3igjS+C1wtIv1xUW6NeR1InjKd/li1iKzETfszB6gGnlXVTTFakqYzoG2tiMwDFuH+N+8CD6hqhT8f+d98F3hMVWf5AsG2jmRIEOeQ8DZdMFoa6YZr0rlKRK7HmdL/+jR9gB3AT3FV7DtV9e8J1hWJhIqEg38Pd0OfKiIH4SLJPhWRfXHhrltxfQDjcW3+31HV5c0YSKL0DsT1M/xLVZd6XaKqn4RBo785jVLV1/3+qcASVZ0tIkfgBgq+jSsNDsQZxPXAOkhKn2KklH+tqt7i908HFnszCIVGr6sM1/8wVVU3ehNYEQadzfxvvgP8VVU/9Pm5SFXnpFpnjOYyXF/sc77GGab87IPr6/zE718GvKWq831+/hzXdLcPLlJ0PS6kPFJQSe4NPQEk3ZSaFSFyIi4i5ixcbeRZYCOu1tID90VEbiCf+ddEovSGAajq53HS0vgn8/uTva4NuL6R7+M6SA/EBXJsxf1oVgRek9DqdDMaj8KFh4dCYzP6zgaOBV7HTel/D25QY6nfXg18rKobk6GvBc3n4GZQCJXGFvLyOFyfZ5h1no77TYZdZ2jyM9D0Hrm3fRM4Chcd+hlu4Hwu7n/+F1y3xxJV3ZFobUlFU9SZxa4di4cDxYH9b+Kq0D/EBSF8ETf4MnK+D+7H9A9ciSYR88Hl4Do7v4xrJjsGF/H2Q+DfMWmPwYVpdw8cS/j0MLhQ9QdxfWGh0+jz8P9wg1XB9d/cB1wBvBiT9mxc1Fa3JOdh6DWmqc6j00Rn6PLTa3saH6QE/C9wPy6S8G8xab+FM9Ck52EiHokOCW8R9TkX6JB8W1UrxIe4qurvcdFGPYE5wBZV/YOIZIkbKPoArlqdh2vWmJMAjZGZHJ5WN6I6H9fx+TugSkQO8J8hV10zVSXwHxH5H//6SF9YwvJZXR/IFar6j/ZqjHwHCdJXA5yvqm/6fNiJGyD9tJPlaroikq+qT+GaI/4jIt+J6Au+n29y63Qa01TnG2miM3T56bWdo2728Uxcje3/cH1NBeKG0iBufNXvcGPc3gxqi/yvxQ1xSPTwlfiRalds6UHA6XHTlhyBa9/9I66JqhDXproZGJ2s0gHRSWZvBc7z2ycAj+ImfOyJM4R3gJNjXjsQ15b9hTBqxDVbfA03HdQpif5ucSW/E/z2V3HRThNi9J0SeF0XXCf0f4ETk/H7C7NG09n5dBKt3R3s968BXojR9m6Mtv1wrSkfRj5TmB8pF7CbL0BoOuZgAC6q61V/7gng9/7cLjNm+zQJmQ0Y19z4P4H9M/yP4T5cP9j+uBLXCf6HdAiuXfh/cct3vAT0SnD+tUXjU7hp/TNxfXh/x81ePAPXgZqwGZ+9vgsC+6fH6JuIG2d1eOQ7xo3niIyH6ZeE3+DeauyfaI3pkpemM27aDqNpYfK0GG0H4IbejPPnu/rn44GZwLHJyMN2f75UC2jnl3INbqBZNdGBha0u40Dy2qmvw40NCprBsbjOym8Gjt2Mi7JJRf410Uh0mv8biA6CHYEzzrwkaZLA9ndxoeRX+/2swLmJuBrgxSnIt3Zr9DetZP0G9zovSVBhLp75mQ7fewq0fYqbrLcxj2hasP8hgb75MD5S1qfUHgL9TXfj5oP6HIgMctVAujEicoMPp4zMITVd3ODdRGmLTFN/J66Jro+IHCEiPXGBEgfgVo+9xKe7RVXXJLI/pw0a+/nowkJxK2IeDFwlIjfgZt1YpX4290SjqhrQdxduFosCETlQ/aS94uaB+wLuBv9G4HP1E5FTxc3FFiaNkXD4XFzJ+00RuTyRGvdSZ2bwPcKq05MjIqUSx6Un4qgz+NvcR0SuE5FLk6TtCEBFZKRGp02LhOzvj4vKjWgbISIXR+6TYSEUIeF7ig83/bOqlvj9HPWDy8TN+HsWbqqbYbjq6om4QXK/TIK22Gnqx+FKT/8CXsT1if1K/fidVNCMxlv95vM4s38IV/1/RgPr/KRKX+D4ibhBwW+q6jP+2ABcBOa7uMl/F+MCP7alWOMbqvrPSFpciPEEXBPpKuAaDYTyplBnMC8jIcmjgTNxzT2XqJsxPzQ6/fFHcdOTjcVFnV6hqh+EQGfwey/E9TE9hhueUe91lse+PlHaRCQPZ5ZH4yb+/UBVbxU39vJ+4N+4AtOnuN9k0v/vsSR0RodEoW5A6/4APqrkdHETL76DG/R6raq+5s+/jmvmu9PvJzSOXwNRL760Ocxv/slffwsu4KG5mRoaae1cHDUK7jfQFfhU/VpW4mZ336RuWpPWNCYkL2PzUKJjSybjIqDe9tcfjpv8dbqqfscfewfX4ZtQU2qDxre8HvF5tMY/XhG3HEFP3ADxhNLWvPSM9v+ni3AzC/TCdeAnnD3IzwJc6PZAVT3KH7sIGC8iH2sCZ/puo85gfo4CylX1J17nP3Fz8SXElLTp8h1H4vrBBuJmzHkCuEdVF4vIfsC5wBO+1QT/m+yBHyCcStKq+S6IuqUmCoGrcWHhS3EdgHOIzmWXiSs93xP5MWk0TDthyyiox+/uwC12FhmIuwo3pb8/1OKsxAeLm/4/kRobfMlI8NV634y3kOgaOY2/EXGrnwabS871tdZE6VMR6QK8482mCHhKVTeIm0V5Im7C1994fYfgJjBNynIJu9G4MWLo4pudvcYLcN9/QpcYaaPOv/u8LPTf4w+AWtxEoKtw42HiPtSiHTqfUj/dDm52kiuBdSJyubjZ8R/2WhNqSG3Q+Xd1w1oyxC0vczquifw3voksssZcIrVFBgaPxEXS1qnqMar6B29Ihbia03acUSEiZwIbVTXlhgRpbEqeelyE282qugi3pHCGRhcOvAM3k8EzItIVeNXfFBpLPJLAMUSeaUA3cXOY/T/cGKL3vIYGX6rZD1cSvFFEZorICbjBsEckWFuEW3Ft5B/hgh3+A8z3GutFJE/culIvAVeKyPvixoodRNMlNeKOunn7zgGW4JYdOMafqscVQmap6ufi+m0iU/3nJVJTWzUGCiZZ4voWfgjchpvlubq590qizotxTTrgpqz5MVCrqk/j7guH4JqaEzrOrg06g985uEH1K3AGegh+uRFNwuq2sTSjM5ifP8KtizYZv9Ixbs2spKzXpap/wBn4dv9/PdOf6ooryH2iqqv8fXEc8K6vhaYeDUG0RTweuD9SL9wiaA/iQq+rgPH+fE9cm+r/4fp2Dgy8NhmRRuOIjs7ujftTFeLCSB8GfuTPnQL8FXiO6PIUiQzLDkbmjCK6xMhAXIk5B3fzvxe43J87B1cYuJfoMgAJiSyL0Xcc8GO/3R9XCOnh9w/CNdFe3sp7JUPj8cCNfnssLiLqOVyY8I9pYUmHwOtzkvRdHxf5zfn9ybgC1HvAy8C5/niz/41E5WVrOnH9cp/iVpUF1xfye9zUOy3pTGZ+3hzYnk40qvUbwB3JzM8YbYfgBgmDq0HNwy97gZsp5y5ixlS29n6JfiTlIsl++JvoO8C9fr8UN+7mFty0PMfhaiaPRW5qwYz3N7zucdKyy1gpf917/PaZBKYDwgVlfEJgjIQ/nsibQBONuJrGP4G7/f6lwJ+IDiy8wt/Ajk+GxhbysB9uTFUmbvzaHbiF+yImleOfc3Gz0/eMaMQNKdhlDZ0452EBLlqsPnJDiEnfGCLuX9sncrP1x76P68+Ja4GphbwMhjU/hgt2iazplBXQmZS8bEVnPq5zPvIdn44rHEXWSwpLfh4LvOK3uwDf9r/PiEllJSM/W9A2HjcDDrjAm7/6/3NBjLbuuEG3xX5/BG78Yk688zD2kZaBDi0R6eRT1XfEzfabDaCq68WFhf8MV4K5U1WPEJGrcEEST6hrjx4kLprrDmBfESlT1Q17o0n9NxrQONRvXuef9wfe9Od64gx0IfC478g/Wl17cOOURdp0OYC9JlYjrrlhtqreFND4ibqmvCG4P9rbwGs+n8er6v2J0hjRFwy6UNf0sBAXbfc5Lg8f1OiyDpEooudwN9mDROS7XvvV6oYVxI1Yjaq6w/e/XQtc639LvwykD+bPA7gIsiki8i2cYd2uCYgWbSEv63wf6wSgBBc8MjtyLvDypORlKzp3ishq4G2JLj3xjLrptdBo0E1K8xNXGDlORD7D1ebLgf/T6DIYkTxNaH42879G3Uzo80RkOa7G9IKqPhA4X+fvQ//GBbscICI34/rMTlbVB+OhbXfCO9yDXVcPDU5ZNBR4HGdOo/yxbrj24H8DJ+PWM/lJ5PeWAH2RUrz4636GqyHdhms6OcWfL8A1Rb4LnNnS50tQHkZKTBm4cVb/xTUt/gV4kuiKmQVEpzb5YlBj4D2yE6izH7BvYP8yok2Qt+KWHwHX33S//xynJkFXsPlkGPBbos23pwNj/PYtuD68EtxKpLfjZoE+P9Eam9HcA7gysH95GPKyGZ2DcMYZGTh/EtHZC1KanzStLQ/EtYREWmAuDkt+4mqSAwL7N+NaHApwXR8P++PD/X/+BeAIfyxh3QmqHbT5rqUfS8yN4lhcM14xzgiuAgbjxhPMI9CslwRtJ/kfwmvA/f7YJFxUWS/cctPP4aau3yf2c/nn3ARrPAHXPPYK8D1/7DTc7MQRjVNxy46Mj9H2PM60xsRZU0bMda70f6CJuJLdcmCIP9cdV5t6OOY9Ej1LemwB6Ys4Ez8LVzJeCQwLnH8JNyYnaRpb0P1tXFNeaPKyhfw8F1czClV+NqPz22H5bTaj7Ub/vy3GFdpn4JeBx/WLTgd+lqw8TOgXE8ZHM1/IGFxJ6td+/w3gW3474QEQMVpOidzQ/f51uEkUf4CreZyGW9r8+ECaPJyRvgr8JPbzJUDjkcDQwP61/kcc0Xg6rmnvsED6NcD/4GZgvgPXdh7vtv1c3CDafXEFkEOBfwTOD8NNDxOpoVwIFAbOJ2UyX9zy1ofjaudDgCcD53NwzTmRGtXZydaYLnkZ0Gn5uXfaCnGD5Yf7/S/hQtsj+Xc0buD/IJ/2ikRrS/gXkg4PXEnlV/7mWZ6C68d2RgabALrgakzTgLNj0h2Daxa6H9dk8acUavw1bqLZM2POLQEu9ds9gG8kUGOwo7sEtx7Xt3Bm/jJwlz+fgev4bpzDLPAeiTb1YFNyV1xAzrW46Mx/48bk7E5jMgw0HnmZNJ2Wn3HTNhgX4fgN3BjQF3ArVge1/TeR2hL6RaTDg0DIKK5jbz6BvpEUawv+4cbhms7G+MfVuFmBT8TNWfc0vm8l0TfW3Wj8C76JERfuPsP/sI+NeV0ywvD3wTWLPoLrJ+zq/1i/INqp/0ecoX8hVh+uv2q/BGschxu/9kdcpF4ksuxnuJJ1UOMJMa8djSv9J3QplDjk5YG48UXJWHrC8nPvtU3w2h7HDZ/Jx/VB3e11lwB/aCEPD8VFxbYaYt7q9ROd+enywHX0PeO3k3ZTb4MuidWDm+l3MdFp83+Jiy4KjcaY86fg+ie6JVFTsP/wMPy6Un7/LFwf1z24Zp+JuGbSyPiiAlzt9GFck+MrJGApAnbt4wzOIn92MxqnA7f48z1wsy78xB//KwnqB21HXn4QynfBpQAABQhJREFUyMveuIUlb/B5+RCuGSgRAUSWn/HVNomA8eECnmK1TcPPSu7TjMXVVOvx96c91pCITE/XB9GY/KT2Je3pj8Xvfw9Xs3se1+ab8uWPiQYelNJ0jEgRrhlvSAg0BpsWv4drRoks4REZlPlnAk2huELABSQ46qiNGiMDqo/A90vg+uj+CkwMUV5Gxt6cGdBZiOtbServwPIz4dryY9KeBfwnsL9Hhfx0n2YoLgQmWKzwz5paRbui0ennM/z+HcB5uFkYRgGDkz0dTCwaHXszBHhdRCLLilyOM86ETpLaFlSbTPV/B24esENE5HpV3eSnUzoDWCsiZ/l0P8cNOKxr8Y0Tr/FgEfmJqpb7sXSDgEkiciWutC24UmrSaEHnoSJytbq5/0bgBl0e4scLnYNbcnxoi2+aPJ2Wn3uv7SARuUhVd0bSiUhvXL/T7YGXZ4pIfxE5uS3XSsulK4zG2ZK/jgt2+BmwXBO8XMOeICIH4sZdZeCizm7ATZuflEkzd4cviEjA7LPUDRycjetk/i2uXf0l/ycMg8YCdYNyX8Y1KX6C64NYhWv/fzZ4g0ihzkxcH8Q9uE78mbiw7Tm4fpLpmpolUSw/46MtQ3ddIiNT3eD6W4GRqnpu4Nwvca0lE3Am+g1VXdLSNTrUjA6dCf9n+gxYqKqzJYGznu8pfnT7h8AJIjIKWKeqm1KtK4ivDTcuPeANaSJQoapfARCRnwMn+rxtSHYNOqjR7+4QkS8Do1X1BK/xRmCKqj7p9xO25EkbdeJvTifg+uF+73U9hBsQ+lYytbWi0/Kz/drqvTbx2rJVtdb/18/E9d/hdV4DTMENXl4hIg/jWlLMlDoiqvpOYDtha0TtKb6qH7nZz0+1ntbQptP9LAI+jdSacGvfjE113sZoXIfrbI7Qn8Bqoqlseo7RuRk3XRYi0g035dfwVOiKxfJz74nRdruIVOKaQB9X1XlAZMXlHwOnqeoKn7YcF+XY4iKnZkpGQtA4z8+XJKpwf/Z3ReQDXF/dryExcw62k48AxC1q+TGujyGyUFtYNIIbzH2uiLyPixRtwI2nCxuWn3vPnbhw8S/hxl5F+DnwmqpODxz7CvBVaLkWan1KhhGDuMUChwJvq+qyVOtpDq9xJG69sMWp1tMS4pbd7oeLxgpVE24Qy89262k0bhE5BjcbxW9w4z0fAn6jqtP8+V/j5v47tbVmUTMlw/AEojBD+6dIB41gOuNNmHXGagv0Md0FvK6qz3vTfwq3nlh5a6ZkzXeG4QnjHz6WwB8/6R3we0KYtQWx/Nx7msnDyNCJZcCd4lb73o4bqFy+u2ZRqykZhmEYCUFEIkvB/6OtzY1mSoZhGEbcia0RtbU2aqZkGIZhhAabZsgwDMMIDWZKhmEYRmgwUzIMwzBCg5mSYRiGERrMlAzDMIzQYKZkGIZhhAYzJcMwDCM0mCkZxh4gIgeKyCwRyRORAhGZLSLjU63LMDoKNnjWMPYQEbkNt5puPlCubrl0wzDigJmSYewhIpIDfIhbf+mQVC8CaBgdCWu+M4w9pyfQFSjE1ZgMw4gTVlMyjD1ERKYCTwJDgL6qelWKJRlGh8HWUzKMPcCvDVOnqo+LSCbwnogco6qvp1qbYXQErKZkGIZhhAbrUzIMwzBCg5mSYRiGERrMlAzDMIzQYKZkGIZhhAYzJcMwDCM0mCkZhmEYocFMyTAMwwgNZkqGYRhGaPj/R/J8uJy01s4AAAAASUVORK5CYII=\n" }, "metadata": { "needs_background": "light" @@ -471,39 +451,69 @@ } ], "source": [ - "# Initialize the non-pretrained version of the model used for this run\n", - "scratch_model = model_ft\n", - "scratch_model = scratch_model.to(device)\n", - "scratch_optimizer = optim.SGD(scratch_model.parameters(), lr=0.001, momentum=0.9)\n", - "scratch_criterion = nn.CrossEntropyLoss()\n", - "_,scratch_hist = train_model(scratch_model, dataloaders_dict, scratch_criterion, scratch_optimizer, num_epochs=num_epochs)\n", + "import scipy.special as sc \n", + "import pandas as pd\n", "\n", - "# Plot the training curves of validation accuracy vs. number\n", - "# of training epochs for the transfer learning method and\n", - "# the model trained from scratch\n", - "ohist = []\n", - "shist = []\n", + "def f(n:int, d:int, p:float):\n", + " a = 0\n", + " for i in range(d, n):\n", + " a += sc.comb(n, i) * (p ** (n - i)) * ((1 - p) ** i) \n", + " return a \n", "\n", - "ohist = [h.cpu().numpy() for h in hist]\n", - "shist = [h.cpu().numpy() for h in scratch_hist]\n", + "n = [i for i in range(8, 42, 2)]\n", + "k = [i for i in range(4, 21)]\n", + "d = [4, 4, 4, 4, 5, 6, 6, 7, 8, 7, 8, 8, 8, 8, 8, 9, 10]\n", + "nkd = list(map(lambda x: \"n={},k={},d={}\".format(x[0], x[1], x[2]), zip(n, k, d)))\n", "\n", - "plt.title(\"Validation Accuracy vs. Number of Training Epochs\")\n", - "plt.xlabel(\"Training Epochs\")\n", - "plt.ylabel(\"Validation Accuracy\")\n", - "plt.plot(range(1,num_epochs+1),ohist,label=\"Pretrained\")\n", - "plt.plot(range(1,num_epochs+1),shist,label=\"Scratch\")\n", - "plt.ylim((0,1.))\n", - "plt.xticks(np.arange(1, num_epochs+1, 1.0))\n", - "plt.legend()\n", - "plt.show()\n" - ] + "composed = {'x' : nkd}\n", + "probs = [0.1, 0.01, 0.001]\n", + "\n", + "indexes = ['x']\n", + "indexes.extend('p={}'.format(p) for p in probs)\n", + "\n", + "for p in probs:\n", + " ans = []\n", + " for item in range(len(n)):\n", + " res = f(n[item], d[item], p)\n", + " ans.append(res)\n", + " composed['p={}'.format(p)] = ans\n", + " print(ans)\n", + " \n", + "df = pd.DataFrame(composed)\n", + "df = df[indexes]\n", + "df.set_index('x', inplace=True)\n", + "df.plot(rot=-30)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": false + } + } }, { "cell_type": "code", "execution_count": null, - "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import scipy.integrate as integrate\n", + "import pandas as pd\n", + "\n", + "n = [i for i in range(8, 42, 2)]\n", + "k = [i for i in range(4, 21)]\n", + "d = [4, 4, 4, 4, 5, 6, 6, 7, 8, 7, 8, 8, 8, 8, 8, 9, 10]\n", + "nkd = list(map(lambda x: \"n={},k={},d={}\".format(x[0], x[1], x[2]), zip(n, k, d)))\n", + "\n", + "composed = {'x' : nkd}\n", + "probs = [0.1, 0.01, 0.001]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } } ], "metadata": { @@ -527,13 +537,13 @@ "pycharm": { "stem_cell": { "cell_type": "raw", + "source": [], "metadata": { "collapsed": false - }, - "source": [] + } } } }, "nbformat": 4, "nbformat_minor": 1 -} +} \ No newline at end of file diff --git a/utilits.py b/utilits.py new file mode 100644 index 0000000..872dfd2 --- /dev/null +++ b/utilits.py @@ -0,0 +1,33 @@ +# load images + +import torch +from torchvision import datasets, transforms + +import os + + +def image_data(dir_path: str, batch_size: int): + input_size = 224 + + # Data augmentation and normalization for training + # Just normalization for validation + composites = { + 'train': transforms.Compose([ + transforms.RandomResizedCrop(input_size), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) + ]), + 'val': transforms.Compose([ + transforms.Resize(input_size), + transforms.CenterCrop(input_size), + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) + ]), + } + # datasets.DatasetFolder(os.path.join(dir_path, 'train'), datasets.folder.default_loader, datasets.folder.IMG_EXTENSIONS) + image_data_sets = {x: datasets.ImageFolder(os.path.join(dir_path, x), composites[x]) for x in ['train', 'val']} + return {x: len(image_data_sets[x]) for x in ['train', 'val']}, { + x: torch.utils.data.DataLoader(image_data_sets[x], batch_size=batch_size, shuffle=True, num_workers=2) for x + in ['train', 'val']} + diff --git a/utils.py b/utils.py deleted file mode 100644 index adffd6d..0000000 --- a/utils.py +++ /dev/null @@ -1,2 +0,0 @@ -# load images - From 96d4aadd04b5510a78691f647f038e08417f4b26 Mon Sep 17 00:00:00 2001 From: glcanvas Date: Mon, 4 Nov 2019 13:46:49 +0300 Subject: [PATCH 03/10] isic image loader --- README.md | 8 +- image_loader.py | 130 ++++++++++++ main.py | 1 + notebooks/init.ipynb | 496 +++---------------------------------------- properties.py | 23 ++ requirements.txt | 4 + utilits.py | 2 +- 7 files changed, 190 insertions(+), 474 deletions(-) create mode 100644 image_loader.py create mode 100644 properties.py diff --git a/README.md b/README.md index ee064f3..e9f6e55 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -такс, ну я пытаюсь ставить pytorch, чорт как же он много весит, хорошо хоть данные на работе скачал +такс, тут выяснилось (хорошо, что заранее) что у одной картинки может быть несколько ответов +поэтому давайте для каждой тренировчной картинки сделаем набор из лэйблов +затем оставим только те лэйблы у которых лэйбл-карта не 0 - -torch.cuda.is_available() -- false АЛЛО ЧИНИТЬ!!! \ No newline at end of file +но как тогда говорить верный ли ответ ? +2^5 вариантов выхода ? -- sounds good \ No newline at end of file diff --git a/image_loader.py b/image_loader.py new file mode 100644 index 0000000..ccb9176 --- /dev/null +++ b/image_loader.py @@ -0,0 +1,130 @@ +from sklearn.utils import shuffle +import torch.utils.data.dataset +import os +from torchvision import transforms +from PIL import Image +import torch +from multiprocessing.pool import ThreadPool +from properties import classes, field_id, field_train, image_size, num_workers + +name_delimiter = '_.' +prefix = 'ISIC_' +attribute = '_attribute_' + +composite = transforms.Compose([ + transforms.Resize(image_size), + transforms.ToTensor(), +]) + + +def images_filter(dir_path: str, suffix: str): + arrays = [] + for _, _, f in os.walk(dir_path): + for file in f: + if not file.endswith(suffix): + continue + id = file.split(suffix)[0].split(prefix)[1] + arrays.append((id, os.path.join(dir_path, file))) + return arrays + + +def map_inputs_labels(data_inputs_path: str, data_labels_path: str): + """ + composed inputs images and labels images to dict with attributes + :return: dict> + """ + inputs = images_filter(data_inputs_path, '.jpg') + labels = images_filter(data_labels_path, '.png') + # + labels_generator = map(lambda x: (x[0].split(attribute)[0], x[0].split(attribute)[1], x[1]), labels) + + labels_dict = dict() + for ids, attr, path in labels_generator: + if ids in labels_dict: + labels_dict[ids][attr] = path + else: + labels_dict[ids] = {attr: path} + + result = [] + for ids, path in inputs: + labels_dict[ids]['id'] = ids + labels_dict[ids]['train'] = path + result.append(labels_dict[ids]) + return result + + +def split_set(values: list, test_size: int, validate_size: int): + """ + get test set, validate set from list of images, previous shuffled + :param values: result of execute map_inputs_label + :param test_size: train set size + :param validate_size: validate set size + :return: train set, validate set + """ + shuffled = shuffle(values) + return shuffled[0:test_size], shuffled[test_size:test_size + validate_size] + + +def map_cell(cell: dict): + for i in classes: + cell[i] = composite(Image.open(cell[i])) + cell[i] = cell[i].to + cell[field_train] = composite(Image.open(cell[field_train])) + return cell + + +def prepare_data(*args): + pool = ThreadPool(num_workers) + for array in args: + for v in array: + pool.apply(map_cell, args=(v,)) + pool.close() + pool.join() + + +def save_data(path: str, data: dict): + ids = data[field_id] + os.makedirs(os.path.join(path, ids), exist_ok=True) + for i in classes: + torch.save(data[i], os.path.join(path, ids, i)) + torch.save(data[field_train], os.path.join(path, ids, field_train)) + + +# either train_path or test_path +def load_tensors(path: str): + result = [] + for tensors in os.listdir(path): + tensor_dir = os.path.join(path, tensors) + if not os.path.isdir(tensor_dir): + continue + res = dict() + res[field_id] = tensors + for fl in os.listdir(tensor_dir): + tensor_file = os.path.join(tensor_dir, fl) + if not os.path.isfile(tensor_file): + continue + res[fl] = torch.load(tensor_file) + result.append(res) + return result + + +def remove_empty_mask(cell: dict): + for c in classes: + if cell[c].sum() > 1: + cell.pop(c) + return cell + + +class CustomDataset(torch.utils.data.dataset.Dataset): + + def __init__(self, data: list): + self.data = data + + def __len__(self): + return len(self.data) + + def __getitem__(self, item): + return self.data[item] + + def __iter__(self): + return iter(self.data) diff --git a/main.py b/main.py index e69de29..3b1ab89 100644 --- a/main.py +++ b/main.py @@ -0,0 +1 @@ +print("sss") \ No newline at end of file diff --git a/notebooks/init.ipynb b/notebooks/init.ipynb index 179e2f8..4fb4ead 100644 --- a/notebooks/init.ipynb +++ b/notebooks/init.ipynb @@ -3,163 +3,37 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "pycharm": { - "is_executing": false, - "name": "#%%\n" - } - }, "outputs": [], "source": [ + "from sklearn.utils import shuffle\n", + "from torchvision import datasets\n", + "import torch.utils.data.dataset\n", + "import os\n", + "from torchvision import transforms\n", + "from PIL import Image\n", "import torch\n", - "import torch.nn as nn\n", - "from torch.nn.modules import loss\n", - "import torch.optim as optim\n", - "import numpy as np\n", - "import torchvision\n", - "from torch.optim.optimizer import Optimizer\n", - "from torchvision import datasets, models, transforms\n", - "import matplotlib.pyplot as plt\n", - "import copy\n", + "from multiprocessing.pool import ThreadPool\n", "\n", - "import utilits" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "pycharm": { - "is_executing": false, - "name": "#%%\n" - } - }, - "outputs": [ - { - "name": "stdout", - "text": [ - "False\n" - ], - "output_type": "stream" - } - ], - "source": [ - "data_dir = \"/home/nikita/PycharmProjects/hymenoptera_data\"\n", - "num_classes = 2\n", - "batch_size = 8\n", - "num_epochs = 25\n", - "print(torch.cuda.is_available())\n", - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "outputs": [], - "source": [ - "def set_parameter_requires_grad(model, feature_extracting):\n", - " if feature_extracting:\n", - " for param in model.parameters():\n", - " param.requires_grad = False" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } - }, - { - "cell_type": "code", - "execution_count": 6, - "outputs": [], - "source": [ - "model_ft = models.alexnet(pretrained=True)\n", + "import image_loader as il\n", + "import properties as pr\n", + "\"\"\"\n", + "r = il.map_inputs_labels(il.data_inputs_path, il.data_labels_path)\n", "\n", - "set_parameter_requires_grad(model_ft, feature_extracting=False)\n", + "train, validate = il.split_set(r, il.train_size, il.test_size)\n", "\n", - "params_to_update = []\n", - "for name,param in model_ft.named_parameters():\n", - " if param.requires_grad:\n", - " params_to_update.append(param)\n", + "il.prepare_data(train, validate)\n", "\n", - "optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)\n", - "criterion = nn.CrossEntropyLoss()\n", + "for i in train:\n", + " il.save_data(il.tensor_train_path, i)\n", + "for j in validate:\n", + " il.save_data(il.tensor_validate_path, j)\n", + "\"\"\"\n", "\n", - "# replace last layer from alexNet to custom classification\n", - "num_ftrs = model_ft.classifier[6].in_features\n", - "model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)\n", + "train_tensor = il.load_tensors(pr.tensor_train_path)\n", + "validate_tensor = il.load_tensors(pr.tensor_validate_path)\n", "\n", - "state_sizes, dataloaders_dict = utilits.image_data(data_dir, batch_size)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } - }, - { - "cell_type": "code", - "execution_count": 7, - "outputs": [], - "source": [ - "def train_model_single_epoch(model:nn.Module, image_data:dict, criterion:nn.CrossEntropyLoss, optimizer:Optimizer,\n", - " epoch:int, is_grad = True):\n", - " state = 'train'\n", - " model.train()\n", - " for inputs, labels in image_data[state]:\n", - " inputs = inputs.to(device)\n", - " labels = labels.to(device)\n", - " \n", - " # flush evaluated gradients\n", - " # what different ?\n", - " optimizer.zero_grad()\n", - " #model.zero_grad()\n", - " \n", - " output = model(inputs)\n", - " loss = criterion(output, labels)\n", - " if is_grad:\n", - " loss.backward()\n", - " # why bellow command ?\n", - " optimizer.step() \n", - " \n", - " return model" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } - }, - { - "cell_type": "code", - "execution_count": 8, - "outputs": [], - "source": [ - "def validate_model_single_epoch(model:nn.Module, image_data:dict, criterion:nn.CrossEntropyLoss, epoch:int):\n", - " state = 'val'\n", - " model.eval()\n", - " sum_loss = 0\n", - " correct_accumulate = 0\n", - " for inputs, labels in image_data[state]:\n", - " inputs = inputs.to(device)\n", - " labels = labels.to(device)\n", - " output = model(inputs)\n", - " loss = criterion(output, labels)\n", - " model_answer = torch.max(output, 1)[1]\n", - " corrects = batch_size - ((model_answer + labels) % 2).sum().item()\n", - " sum_loss += loss.item() * batch_size\n", - " correct_accumulate += corrects\n", - " \n", - " epoch_loss = sum_loss / state_sizes[state]\n", - " epoch_acc = float(correct_accumulate) / state_sizes[state]\n", - " print(\"epoch: {}, corrects:{:.4f}, loss:{:.4f}\".format(epoch, epoch_acc, epoch_loss))\n", - " return epoch_acc" + "train_tensor = list(map(il.remove_empty_mask, train_tensor))\n", + "validate_tensor = list(map(il.remove_empty_mask, validate_tensor))" ], "metadata": { "collapsed": false, @@ -171,318 +45,11 @@ }, { "cell_type": "code", - "execution_count": 9, - "outputs": [], - "source": [ - "def train_model(model, image_data, criterion, optimizer, num_epochs=25, is_grad=True):\n", - " best_model_state = copy.deepcopy(model.state_dict())\n", - " best_acc = 0.0\n", - " for epoch in range(num_epochs):\n", - " print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n", - " print('-' * 10)\n", - " m = train_model_single_epoch(model, image_data, criterion, optimizer, epoch, is_grad)\n", - " acc = validate_model_single_epoch(m, image_data, criterion, epoch)\n", - " if acc > best_acc:\n", - " best_acc = acc\n", - " best_model_state = copy.deepcopy(m.state_dict())\n", - " \n", - " best_model = model.load_state_dict(best_model_state)\n", - " print(best_acc)\n", - " print(best_model)\n", - " return best_model" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "pycharm": { - "is_executing": false, - "name": "#%%\n" - } - }, - "outputs": [ - { - "name": "stdout", - "text": [ - "Epoch 0/24\n----------\n", - "epoch: 0, corrects:0.8497, loss:0.4499\nEpoch 1/24\n----------\n", - "epoch: 1, corrects:0.9216, loss:0.4158\nEpoch 2/24\n----------\n", - "epoch: 2, corrects:0.9150, loss:0.3640\nEpoch 3/24\n----------\n", - "epoch: 3, corrects:0.9020, loss:0.4004\nEpoch 4/24\n----------\n", - "epoch: 4, corrects:0.9216, loss:0.3876\nEpoch 5/24\n----------\n", - "epoch: 5, corrects:0.9477, loss:0.3656\nEpoch 6/24\n----------\n", - "epoch: 6, corrects:0.9085, loss:0.5285\nEpoch 7/24\n----------\n", - "epoch: 7, corrects:0.9477, loss:0.3414\nEpoch 8/24\n----------\n", - "epoch: 8, corrects:0.9412, loss:0.3912\nEpoch 9/24\n----------\n", - "epoch: 9, corrects:0.9281, loss:0.3954\nEpoch 10/24\n----------\n", - "epoch: 10, corrects:0.9281, loss:0.3575\nEpoch 11/24\n----------\n", - "epoch: 11, corrects:0.9608, loss:0.3822\nEpoch 12/24\n----------\n", - "epoch: 12, corrects:0.9412, loss:0.3862\nEpoch 13/24\n----------\n", - "epoch: 13, corrects:0.9346, loss:0.4379\nEpoch 14/24\n----------\n", - "epoch: 14, corrects:0.9150, loss:0.5164\nEpoch 15/24\n----------\n", - "epoch: 15, corrects:0.9281, loss:0.4536\nEpoch 16/24\n----------\n", - "epoch: 16, corrects:0.9281, loss:0.4430\nEpoch 17/24\n----------\n", - "epoch: 17, corrects:0.9412, loss:0.4107\nEpoch 18/24\n----------\n", - "epoch: 18, corrects:0.9477, loss:0.3834\nEpoch 19/24\n----------\n", - "epoch: 19, corrects:0.9477, loss:0.4825\nEpoch 20/24\n----------\n", - "epoch: 20, corrects:0.9216, loss:0.4024\nEpoch 21/24\n----------\n", - "epoch: 21, corrects:0.9020, loss:0.5813\nEpoch 22/24\n----------\n", - "epoch: 22, corrects:0.9346, loss:0.3711\nEpoch 23/24\n----------\n", - "epoch: 23, corrects:0.9477, loss:0.3916\nEpoch 24/24\n----------\n", - "epoch: 24, corrects:0.9412, loss:0.3231\n0.9607843137254902\n\n" - ], - "output_type": "stream" - } - ], - "source": [ - "trained_model = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "pycharm": { - "is_executing": false, - "name": "#%%\n" - } - }, + "execution_count": 2, "outputs": [], "source": [ - "model_conv = models.alexnet(pretrained=True)\n", - "num_ftrs = model_conv.classifier[6].in_features\n", - "model_conv.classifier[6] = nn.Linear(num_ftrs,num_classes)\n", - "\n", - "params = []\n", - "for param in model_conv.parameters():\n", - " param.requires_grad = False\n", - " params.append(param)\n", - "optimizer_conv = optim.SGD(params, lr=0.001, momentum=0.9)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "outputs": [ - { - "name": "stdout", - "text": [ - "Epoch 0/14\n----------\n", - "epoch: 0, corrects:0.5752, loss:0.7454\nEpoch 1/14\n----------\n", - "epoch: 1, corrects:0.5752, loss:0.6931\nEpoch 2/14\n----------\n", - "epoch: 2, corrects:0.5752, loss:0.7215\nEpoch 3/14\n----------\n", - "epoch: 3, corrects:0.5752, loss:0.6915\nEpoch 4/14\n----------\n", - "epoch: 4, corrects:0.5752, loss:0.6903\nEpoch 5/14\n----------\n", - "epoch: 5, corrects:0.5752, loss:0.7229\nEpoch 6/14\n----------\n", - "epoch: 6, corrects:0.5752, loss:0.6956\nEpoch 7/14\n----------\n", - "epoch: 7, corrects:0.5752, loss:0.6935\nEpoch 8/14\n----------\n", - "epoch: 8, corrects:0.5752, loss:0.7454\nEpoch 9/14\n----------\n", - "epoch: 9, corrects:0.5752, loss:0.7066\nEpoch 10/14\n----------\n", - "epoch: 10, corrects:0.5752, loss:0.6941\nEpoch 11/14\n----------\n", - "epoch: 11, corrects:0.5752, loss:0.7106\nEpoch 12/14\n----------\n", - "epoch: 12, corrects:0.5752, loss:0.7062\nEpoch 13/14\n----------\n", - "epoch: 13, corrects:0.5752, loss:0.7438\nEpoch 14/14\n----------\n", - "epoch: 14, corrects:0.5752, loss:0.7209\n0.5751633986928104\n\n" - ], - "output_type": "stream" - } - ], - "source": [ - "trained_model_conv = train_model(model_conv, dataloaders_dict, criterion, optimizer_conv, num_epochs=15, is_grad=False)\n", - "\n", - "\n", - "\n", - "\"\"\"\"\n", - "\n", - "d) \n", - "H =\n", - "1 1 1 1 1 1\n", - "0 1 0 1 1 0\n", - "0 0 1 0 1 1\n", - "=>\n", - "H = \n", - "1 1 0 1 0 0\n", - "1 0 0 0 1 0\n", - "1 0 1 0 0 1\n", - "G = \n", - "1 0 0 1 1 1\n", - "0 1 0 1 0 0\n", - "0 0 1 0 0 1\n", - "=> r = 3, k = 3, n = 6, d = 1\n", - " \n", - "e) \n", - "G = \n", - "0 0 1 0 1 1\n", - "1 1 1 1 1 1\n", - "1 0 1 1 0 0\n", - "=> \n", - "G = \n", - "0 0 1 1 0 0\n", - "1 1 1 0 1 0\n", - "1 1 1 0 0 1\n", - "H = \n", - "0 0 1 1 0 0\n", - "1 1 1 0 1 0\n", - "1 1 1 0 0 1\n", - "n = 6, k = 3, r = 3, d = 1\n", - "\n", - "f) \n", - "G = \n", - "0 1 0 1 1 0\n", - "1 1 1 1 1 1 \n", - "=> \n", - "G = \n", - "1 0 1 0 0 1\n", - "0 1 0 1 1 0\n", - "H = \n", - "1 0 1 0 0 0 \n", - "0 1 0 1 0 0\n", - "0 1 0 0 1 0\n", - "1 0 0 0 0 1\n", - " ^ ^ ^\n", - "=>\n", - "n = 6, k =2, r = 4, d = 2 \n", - "g)\n", - "H = \n", - "0 1 0 1 1 0 0\n", - "1 0 1 1 0 1 0\n", - "1 1 1 1 1 1 1\n", - " ^ ^\n", - "=> \n", - "H =\n", - "0 1 0 1 1 0 0\n", - "1 0 1 1 0 1 0\n", - "0 0 0 1 0 0 1\n", - "=>\n", - "G =\n", - "1 0 0 0 0 1 0\n", - "0 1 0 0 1 0 0\n", - "0 0 1 0 0 1 0\n", - "0 0 0 1 1 1 1\n", - "=>\n", - "n = 7, k = 4, r = 3, d = 1\n", - "\n", - "h)\n", - "G = \n", - "1 1 1 0 1 0 0\n", - "0 1 1 1 0 1 0\n", - "0 0 1 1 1 0 1\n", - "=> \n", - "G = \n", - "1 0 0 1 1 1 0\n", - "0 1 0 0 1 1 1\n", - "0 0 1 1 1 0 1\n", - "=>\n", - "H = \n", - "1 0 1 1 0 0 0\n", - "1 1 1 0 1 0 0\n", - "1 1 0 0 0 1 0\n", - "0 1 1 0 0 0 1\n", - "^ ^ ^ ^\n", - "=> \n", - "n = 7, k = 3, r = 4, d = 3\n", - "\n", - "i)\n", - "H = \n", - "1 1 1 0 1 0 0\n", - "0 1 1 1 0 1 0\n", - "1 1 1 1 1 1 1\n", - "^ ^\n", - "=> \n", - "H = \n", - "1 1 1 0 1 0 0 \n", - "0 1 1 1 0 1 0\n", - "0 1 1 0 0 0 1\n", - "=> \n", - "G = \n", - "1 0 0 1 0 0 0\n", - "1 1 1 0 1 0 0\n", - "1 1 1 0 0 1 0\n", - "0 1 0 0 0 0 1\n", - "=> \n", - "n = 7, k = 4, r = 3, d = 1\n", - "\n", - "\"\"\"" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } - }, - { - "cell_type": "code", - "execution_count": 80, - "outputs": [ - { - "name": "stdout", - "text": [ - "[0.5691011400000001, 0.6513124383000002, 0.7175702976840004, 0.7712320727218503, 0.8146979798789038, 0.8499053641763854, 0.8784233453999499, 0.9015229097775138, 0.9202335569213986, 0.9353891811077326, 0.9476652366972641, 0.9576088417247846, 0.9656631617970758, 0.9721871610556316, 0.9774716004550617, 0.9817519963686003, 0.9852191170585665]\n[0.0772553001108834, 0.09561792499002673, 0.11361512828387049, 0.1312541872310217, 0.14854222890512433, 0.16548623854991237, 0.18209306240276912, 0.198369410460954, 0.214321859192781, 0.22995685419484466, 0.24528071279636723, 0.2602996266117195, 0.2750196640421463, 0.2894467727277076, 0.3035867819504261, 0.3174454049896126, 0.3310282414303194]\n[0.007972055930000112, 0.009955119790251791, 0.011934219505791075, 0.013909363000999002, 0.015880558184360002, 0.017847812948549473, 0.019811135170465316, 0.021770532711259557, 0.023726013416369752, 0.025677585115550426, 0.02762525562290444, 0.02956903273691425, 0.03150892424047316, 0.03344493790091646, 0.03537708147005253, 0.037305362684193884, 0.039229789264188186]\n" - ], - "output_type": "stream" - }, - { - "data": { - "text/plain": "" - }, - "metadata": {}, - "output_type": "execute_result", - "execution_count": 80 - }, - { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaUAAAExCAYAAADcJb37AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2dd5icVdn/P/f2zWaTbHY3ddN7IUAIhNCr9KLSRAWkKQi8IqgIigq8P1EElapYaAq8IiIBqVKkJUAIGEgPqZueTc9m+/3745zZeXayu9lspjyze3+ua655ypl5vnNm5vmecp9zRFUxDMMwjDCQkWoBhmEYhhHBTMkwDMMIDWZKhmEYRmgwUzIMwzBCg5mSYRiGERrMlAzDMIzQkJVqAa1RUlKigwcPTrUMwzCMtOKjjz7aoKqlqdbRHkJtSoMHD2bGjBmplmEYhpFWiMiyVGtoL9Z8ZxiGYYSGuJmSiPxZRNaJyGctnBcRuVtEFonILBGZGK9rG4ZhGB2DeNaUHgZObOX8ScAI/7gceCCO1zYMwzA6AHEzJVV9C9jYSpIzgEfVMR3oISJ943V9wzAMI/1JZp9Sf2BFYL/cHzMMwzAMILmmJM0c22WKchG5XERmiMiM9evXJ0GWYRiGERaSaUrlwIDAfhmwKjaRqj6oqpNUdVJpaVqG2RuGYRjtJJnjlKYCV4nIk8BkYIuqrk7i9Q3DMFJCQ4NSVVfPzpp6Kmvq2Vkb3a6qrae6rp6q2ga/3fS5qrah8Xzwubq2gaoWntOZuJmSiDwBHAWUiEg58BMgG0BVfwe8AJwMLAIqgW/E69qGYRjxoK6+gR3V9WyvqWNHdR3bq92z265nR3UdO2ujZlJZUxfYdkazM2Y78twesjKEvOxMcrMyGp9zszPJy84gNyuDngU5jefysjLJzXbbN8U5X5JJ3ExJVb+ym/MKfDte1zMMwwBQVapqG9haVcvWnbX+uY6tVbWNphIxlO1VdY2GEzweMaDqurbXMvKzM+mSk0mef87PySQ/O5OeBTmUFUWPd8nJiqbJjqaLvCZiKHnZGbsYUFZm+3pYzJQMwzD2guq6+kYj2bIzYi51TUxmS+O2O7ctcK6mfvdmUpCTSUFuFl1zsyjIzaIgN5P+PfL8tjvetXE7s8nxgpzIOWcyuVkZZGQ0F7tl7C1mSoZhxJXa+gY2VdawaUctG3fUsKmyxj3vqGFTZW10P3B8R03rzVvZmUL3/Gy65WVTmJ9Nt7wsyoryG491y8/yz+5c9/xsCvOyG42kICfLTCRNMFMyDKNV6uobqNhRw7qt1azbVsX6bdVsrHRmsnHHriazraquxffqmptFUUE2PbvkUNQlh2GlXSnqkkNRl2x6dImYSlOT6Z6fTW5WBiJmKp0BMyXD6KTsqK5j/bZq1m2Lms26bdXRY1ur2LC9moodNeguIwpp7D8pKsimqEsOg4q7UNQlxx3rkk1RQY4znwJ3rEeXbHKzMpP/QY20wkzJMDoYNXUNrNlSRfnmSlZtrmLdtirWba1m/fZq1gdqO801mWVlCKWFufQqzKWsqAv7DyyiV2Fu47Fe3fIo6ZpDcUEu+TlmMEb8MVMyjDRCVdm6s67RcFZt3slK/1i1eScrN+1k/fbqXWo2XXOzGs1lfP/u9CrMo1e3XEq75rrnwlx6FebRIz/b+l6MlGKmZBghoq6+gbXbqlm5aecuhhMxndgaTk5WBv175NOvRx5Hjiylf1E+/Xrk+2P59O6WS5cc+6sb6YH9Ug0jydTWN1C+aSdLN+xg8YYdLN2wgyX+sWZrFfUNTas5RV2y6V+Uz+DiAg4ZVkJZjOkUF+RY7cboMJgpGUYCaGhQVm+tYsn6HSyp2MGS9TtYWuGMZ8XGSuoCxlOYl8XQkgImDS5iQFGXQE0nj3498q2WY3Qq7NduGO1EVVm/vZqlGypZsmE7S/zz0g2VLK3Y0WR2gPzsTAaXFDCmbyEn79OHwcUFDC0tYHBxAT0Lcizc2TA8ZkqG0QY2V9Ywd/U25q3Zyjz//Pn6HWyvjo7Jyc4UBvbswpCSrhwxsoQhJV0ZXNKFoSVd6d0t14zHMNqAmZJhBKirb2DJhh3MXbONuau3Mm/1Vuat2cbqLVWNaXoW5DC6TyFfntifISUFDC4pYGhJV/r1yGv3XGWGYTjMlIxOy8YdNcxdvdWZzxpX+1mwdjs1vtktK0MY3qsrBw8tZnSfQkb37caYvoWUdrVaj2EkCjMlo8NTW9/A5+u3M2/1Nuau2eqa4VZvZd226sY0JV1zGdO3kIsOGczoPoWM6duNYaVdycmymo9hJBMzJaPDsW5bFTOXbebj5ZuYuXwTs8q3NAYd5GRmMLxXVw4bUcKYPt0Y07cbo/oUUlqYm2LVhmGAmZKR5tTWNzBv9TZmegOauXwTKzbuBFzgwbh+3fnq5EFMKOvOmL7dGFpaQLb1+xhGaDFTMtKKiu3VzFy+mZnLN/HRsk3MKt9MlV/+uVdhLgcMKuKCgwczcVAPxvXrTl62zc9mGOmEmZIRWurqG5i/dhszl21qNKJlFZWAC0IY168b5x04kImDipg4sAf9e+RbAIJhpDlmSkZo2FlTzwdLN/LBkgpmLtvMf8s3U+nneSvpmsvEgT34ykEDOWBQEfv0t1qQYXREzJSMlKGqzF+7jbcWrOetBRv4YOlGauoayMwQxvQt5OwDynwtqIiyIqsFGUZnwEzJSCoV26t5Z9EG3lqwgbcXrm8Myx7RqytfP3gQR4ws5cDBRTbfm2F0UuyfbySUmroGZi7fxFsL1vP2wg18tmoLqtCjSzaHDS/hiBGlHD6yhL7d81Mt1TCMEGCmZMQVVWVpRaU3ofVM+7yCHTX1ZGYIEwf24LvHjeTwkaXs0787mbbcgmEYMZgpGXvN1qpa3ltUwVsLnRFFxgkN7NmFL07sz+EjSjlkWDGFedkpVmoYRtgxUzLaxYqNlTw3axWvz13Hxys2U9+gFORkMmVYCZcfPpQjRpYyqLgg1TINw0gzzJSMNrNq807+NWs1z89axX/LtwCwT//uXHHkMA4fUcLEQUU2W4JhGHuFmZLRKuu2VvGvT1fz/KzVfLRsE+CM6IcnjeaUCX0pK+qSYoWGYXQkzJSMXajYXs2Ln63h+VmreH/JRlRhdJ9CvnfCKE7Zpy+DS6xZzjCMxGCmZABuZdWXZ6/h+Vmree/zCuoblGGlBVxzzAhO27cvw3sVplqiYRidADOlTszWqlpenb2W52et4p1FG6itVwb27MK3jhzKqRP6MbpPoc2iYBhGUjFT6mRU1tTx77nreP6/q3hzwXpq6hro3yOfbxw6hFMn9GWf/t3NiAzDSBlmSp2A+gblPwvW8fRHK3lt3lqqahvo3S2Xr04eyKkT+rH/gB5k2EBWwzBCQNxMSUROBH4LZAJ/VNXbY84PBB4Bevg0N6jqC/G6vrErmytr+NuMFTw2fRkrNu6kuCCHsw8YwKkT+nLg4J5mRIZhhI64mJKIZAL3AccD5cCHIjJVVecEkv0I+JuqPiAiY4EXgMHxuL7RlM9WbuHRaUt59pNVVNc1cNCQntxw4hi+MK63jSMyDCPUxKumdBCwSFUXA4jIk8AZQNCUFOjmt7sDq+J0bQOorqvnxU/X8Oi0pcxcvpn87Ey+fEAZF0wZxOg+3Xb7esMwjDAQL1PqD6wI7JcDk2PS/BR4RUSuBgqA45p7IxG5HLgcYODAgXGS13FZtXknj7+/nCc/XM6G7TUMKSng5lPH8uUDyuieb3PNGYaRXsTLlJrrnNCY/a8AD6vqnSIyBXhMRMarakOTF6k+CDwIMGnSpNj3MHAzcU9bXMGj7y3j1blraVDl2NG9uGDKYA4bXmJ9RYZhpC3xMqVyYEBgv4xdm+cuAU4EUNVpIpIHlADr4qShw7O9uo5nZpbz6LRlLFy3nR5dsrn08CF8bfIgBvS06X4Mw0h/4mVKHwIjRGQIsBI4Dzg/Js1y4FjgYREZA+QB6+N0/Q7NonXbeWzaUp6euZLt1XXs0787d5w1gdP27Udedmaq5RmGYcSNuJiSqtaJyFXAy7hw7z+r6mwRuQWYoapTgeuAP4jItbimvYtU1ZrnWqCuvoHX5q3j0WlLeXdRBTmZGZwyoS8XTBnEfgN62ABXwzA6JBJmX5g0aZLOmDEj1TKSSlVtPY+/v5w/vbOElZt30q97Hl89eBDnHjiAkq65qZZnGEYaICIfqeqkVOtoDzajQ0iorW/gqRnl3PP6QlZvqWLykJ78+NSxHDemF1k2tsgwjE6CmVKKqW9Qnv1kJb/590KWb6xk4sAe3HnOvhwyrCTV0gzDMJKOmVKKUFVe+mwNd726gIXrtjO2bzf+fNEkjh7Vy/qLDMPotJgpJRlV5c0F67nzlfl8tnIrw0oLuO/8iZw0vo+NLzIMo9NjppREpn1ewZ2vzGfGsk0M6JnPnWfvy5n79yfTzMgwDAMwU0oKHy/fxJ2vLOCdRRvo3S2X284czzmTBpCTZQEMhmEYQcyUEsjc1Vu585X5/HvuOooLcvjRKWP42sGDbMCrYRhGC5gpJYDP12/n168u4PlZqynMy+L6L4zkG4cOoSDXstswDKM17C4ZR1ZsrOTu1xby9Mxy8rIzuero4Vx2+FC6d7HZug3DMNqCmVIcWLu1intfX8STHy5HRPjGoUO44qhhNgODYRjGHmKmtBdU19XzwJuf88Cbn1PfoJxz4ACuPmY4fbvnp1qaYRhGWmKm1E4+WraJG56excJ12zl1Ql++f8JoBhbb8hGGYRh7g5nSHrK9uo47XprHo9OX0bdbHg9ddCBHj+6ValmGYRgdAjOlPeD1eWu56ZnPWLO1igunDOb6E0bR1SLqDMMw4obdUdvAhu3V/Oy5OTz331WM7N2V+756CBMHFqValmEYRofDTKkVVJWnZ67ktn/NobK6nu8eP5JvHTnMZmIwDMNIEGZKLbC8opIbn/mUdxZtYNKgIm7/8j4M71WYalmGYRgdGjOlGOrqG3jo3aXc+ep8sjIyuPXM8Xz1oIE2g7dhGEYSMFMKMHvVFm54+lM+XbmF48b04tYzx9uYI8MwjCRipgRU1dbz29cW8uBbiynqks1950/k5H362GJ7hmEYSabTm9J7n2/gxn98ytKKSs6ZVMaNJ4+hR5ecVMsyDMPolHRaU9pSWcvPX5zLkx+uYFBxF/566WQOHV6SalmGYRidmk5nSqrKS5+t4eaps9m4o4ZvHjmU7xw7kvwcW+PIMAwj1XQqU1q/rZqbnvmUV+asZVy/bjx00YGM79891bIMwzAMT6cxpY+Xb+KKv8xkU2UNPzxpNJccNoSsTBsEaxiGESY6hSk9+cFybn52Nr275/LMlYcytl+3VEsyDMMwmqFDm1J1XT0/nTqHJz5YzuEjSrjnK/tbZJ1hGEaI6bCmtHZrFd/6y0d8vHwzVx41jOu+MIpMm5XBMAwj1HRIU/pw6Uau+MtMKmvqeOCrEzlpn76plmQYhmG0gQ5lSqrKY9OXcctzcxjQswuPXzaZkb1tElXDMIx0ocOYUlVtPTc98xlPzyzn2NG9uOvc/eien51qWYZhGMYeELeYaBE5UUTmi8giEbmhhTTniMgcEZktIo/H69orN+/k7N9N4+mZ5fzPsSP4wwWTzJAMwzDSkLjUlEQkE7gPOB4oBz4UkamqOieQZgTwQ+BQVd0kIr3ice33Pt/AVY9/TG1dA3+4YBLHj+0dj7c1DCOJ1NbWUl5eTlVVVaqlpBV5eXmUlZWRnd1xCuHxar47CFikqosBRORJ4AxgTiDNZcB9qroJQFXX7c0FVZU/vbOEn784jyElBfz+6wcwrLTr3rylYRgpory8nMLCQgYPHmyz87cRVaWiooLy8nKGDBmSajlxI17Nd/2BFYH9cn8syEhgpIi8KyLTReTE9l6ssqaOa578hNv+NZfjx/Tmn98+1AzJMNKYqqoqiouLzZD2ABGhuLi4w9Uu41VTau6XpM1cawRwFFAGvC0i41V1c5M3ErkcuBxg4MCBu7zp8opKLn9sBvPXbuN7J4ziyqOG2Q/ZMDoA9j/eczpinsWrplQODAjslwGrmknzrKrWquoSYD7OpJqgqg+q6iRVnVRaWtrk3H8WrOe0e99h1eadPHTRgXz76OEd8ksxDCN9UFWuueYahg8fzoQJE5g5c2az6W666SYGDBhA167WqtMa8TKlD4ERIjJERHKA84CpMWn+CRwNICIluOa8xW15c1XlvjcWcdFDH9C3ex7PXX0YR42KS5yEYRjGXvHiiy+ycOFCFi5cyIMPPsgVV1zRbLrTTjuNDz74IMnq0o+4mJKq1gFXAS8Dc4G/qepsEblFRE73yV4GKkRkDvAG8D1Vrdjde2+vruOKv8zkjpfnc+qEfvzjykMYVFwQD9mGYRiNLF26lNGjR3PhhRcyYcIEzjrrLCorK3f7umeffZYLLrgAEeHggw9m8+bNrF69epd0Bx98MH372uwyuyNug2dV9QXghZhjNwe2Ffiuf7SJ6roGzrzvXRav386PThnDJYcNseY6w+jg/Oy52cxZtTWu7zm2Xzd+ctq43aabP38+f/rTnzj00EO5+OKLuf/++1m5ciVvvPHGLmnPO+88brjhBlauXMmAAdHei7KyMlauXGkG1E5CPaPDonXbyd5Rw18umcwhtlS5YRgJZsCAARx66KEAfO1rX+Puu+/mn//8Z6uvceXtpljhuf2E2pTyszOZetWhlBV1SbUUwzCSRFtqNIki1kxEhGuvvbbVmlJZWRkrVkRHxJSXl9OvX7+Ea+2ohNqUhpYWmCEZhpE0li9fzrRp05gyZQpPPPEEhx12GNddd12rrzn99NO59957Oe+883j//ffp3r27Nd3tBbYeuGEYhmfMmDE88sgjTJgwgY0bN7YYSRfk5JNPZujQoQwfPpzLLruM+++/v/Hcfvvt17j9/e9/n7KyMiorKykrK+OnP/1pIj5C2iPNtYeGhUmTJumMGTNSLcMwjAQzd+5cxowZk1INS5cu5dRTT+Wzzz5LqY49pbm8E5GPVHVSiiTtFVZTMgzDMEKDmZJhGAYwePDgtKsldUTMlAzDMIzQYKZkGIZhhAYzJcMwDCM0mCkZhmEYocFMyTAMYy9o69IVH330Efvssw/Dhw/nmmuuaZye6KmnnmLcuHFkZGRgQ2DMlAzDMPaKti5dccUVV/Dggw82pn3ppZcAGD9+PP/4xz844ogjkik7tJgpGYZhkNilK1avXs3WrVuZMmUKIsIFF1zQONHrmDFjGDVqVEI+UzoS6rnvDMPohLx4A6z5NL7v2WcfOOn23SZL1NIVK1eupKysbJc0xq6YKRmGYXgStXSFLW/RdsyUDMMIF22o0SSKRC1dUVZWRnl5eatpDIeZkmEYhidRS1f07duXwsJCpk+fzuTJk3n00Ue5+uqrE/lR0hYLdDAMw/AkcumKBx54gEsvvZThw4czbNgwTjrpJACeeeYZysrKmDZtGqeccgonnHBC/D9YGmFLVxiGkXJs6Yr2Y0tXGIZhGEaCMFMyDMPAlq4IC2ZKhmEYRmgwUzIMwzBCg5mSYRiGERrMlAzDMIzQYKZkGIaxF+zt0hUbN27k+OOPZ8SIERx//PFs2rQJgHnz5jFlyhRyc3P51a9+lbTPk2rMlAzDMPaCvV264vbbb+fYY49l4cKFHHvssdx+u5tmqWfPntx9991cf/31SfssYcBMyTAMg9QtXfHss89y4YUXAnDhhRc2Hu/VqxcHHngg2dnZcf6k4cbmvjMMI1T84oNfMG/jvLi+5+ieo/nBQT/YbbpULF2xdu3axrR9+/Zl3bp17f6cHQEzJcMwDI8tXZF64mZKInIi8FsgE/ijqjY7/7yInAU8BRyoqjaxnWEYTWhLjSZRpGLpit69e7N69Wr69u3L6tWr6dWrVzw/UtoRlz4lEckE7gNOAsYCXxGRsc2kKwSuAd6Px3UNwzDiSWTpCqBx6Ypf//rXfPLJJ7s8brjhBsAtXfHoo4+iqkyfPn23S1eoKo8++ihnnHFG4+sfeeQRAB555JHG452VeAU6HAQsUtXFqloDPAk0l7O3Ar8EquJ0XcMwjLiRiqUrbrjhBl599VVGjBjBq6++2mh2a9asoaysjLvuuovbbruNsrIytm7dGudPHD7i1XzXH1gR2C8HJgcTiMj+wABVfV5EOleMo2EYaUFGRga/+93v9ug1IsJ9993X7LlPPvmkcXvSpEnNTvhaXFzMa6+9tsvxPn36NGny6yzEq6bUXI9dY8+eiGQAvwZaX8LRpb1cRGaIyIz169fHSZ5hGIaRDsTLlMqBAYH9MmBVYL8QGA+8KSJLgYOBqSKyyyJUqvqgqk5S1UmlpaVxkmcYhtE6tnRFOIiXKX0IjBCRISKSA5wHTI2cVNUtqlqiqoNVdTAwHTjdou8MwzCMIHExJVWtA64CXgbmAn9T1dkicouInB6PaxiG0bFpbiyP0TodMc/iNk5JVV8AXog5dnMLaY+K13UNw0h/8vLyqKiooLi42AaVthFVpaKigry8vFRLiSs2o4NhGCknMrjUgpv2jLy8vCbTF3UEzJQMw0g52dnZDBkyJNUyjBBgs4QbhmEYocFMyTAMwwgNZkqGYRhGaDBTMgzDMEKDmZJhGIYRGsyUDMMwjNBgIeGGYRjpTM0O2LQMNi2Fzf45jTFTMgzDCDMN9bB1pTObTUt3NaAdMQOOc7omX2McMVMyDMNIJaqwc1PUdCJmEzGgLSugoS6aXjKhexkUDYZRJ7nnosHQwz936Qk3pW/PjJmSYRhGoqmvdeaycUnAcJZEjac6ZkXZLsXOYPrtD+O+6I1nkHvuVgaZHffW3XE/mWEYRjKJ1HaaM54t5aAN0bSZuVGTGXAw9BwCPQZFzSe3MBWfIBSYKRmGYbSF+jrYWh41nFjzqdrSNH1BqTedyTDhXCgaEm1qK+wLGenbxJZIzJQMwzAi1FRGTWbjkqbPm5c37dvJyI7WdsomecMZYrWdvcRMyTCMzoMqVFbEGM7S6Pb2NU3T53V3RtN3Xxh7pmtmixhPt36QkZmKT9GhMVMyDKNjEQmh3rgENi5uWttpLqigsJ8zm+HHQc/BznQi5tOlZ0o+QmfGTMkwjPSjvtY1p21cHDWfiAFtWgr1NdG0mTnRIIKBU5qaTtEgyM5P1acwmsFMyTCMcFK70wcUxBjPxsU+mq0+mja7AHoOhdLRbuxOz6HefIZaM1uaYaZkGEbqqN7W1Gw2Loka0LZVTdPm9XAmU3agi2br6U2n51AX6SaSms9gxBUzJcMwEkvV1oDpfB41nYrPYce6pmkLejmTGXqUNxzr3+lsmCkZhrH3VG31hrMYKoIGtHjXudkK+zrDGXlCtKYTeeSm97xtxt5jpmQYRtuIGE/F59GaT2S7ckPTtIX9nMlE+nd6DovWfHIKUqPfSAvMlAzDiFKzI2A23oAi27E1nm79ndGMPsU9Fw+LBhjkdEmNfiPtMVMyjM5GbZULnW7OeLatbpq2ax9nNiNPdM/Fw32tZ4iFUhsJwUzJMDoikXE8FYv8I2BAW8oBjabtUuIMZ+jRUOyb2oqHWx+PkRLMlAwjXWlocDWboPFEtjcvazpPW153ZzQDp/hmtmFRA8rvkbrPYBgxmCkZRtip3NjUcII1n9rKaLqsfGc4fcbDuDOjTW3Fw104tY3jMdIAMyXDCAM1lb55LWI6i6PbOzdG00mmmy6neDgMOcL383jjKexnyyEYaY+ZkmEki4Z6159TsRA2RMzHb28tb5q2sJ8zm7FnOMOJPIoGQWZ2avQbRhIwUzKMeNPY3LYQNixs2uRWXx1Nl9vNGc2gQ6BkhK/xjLAAA6NTY6ZkGO2hrtpNlxOs7US2Kyui6TKyfHPbCBh2jDefEc6Muvayfh7DiCFupiQiJwK/BTKBP6rq7THnvwtcCtQB64GLVXVZvK5vGHFHFXZs8KazwNV6Nix0+5uWgjZE0xb0coYz+lRnOBHzseY2w9gj4mJKIpIJ3AccD5QDH4rIVFWdE0j2MTBJVStF5Argl8C58bi+YewVdTVuMOkGbz4Vi6ImVLU5mi4rz0Wz9ZkA47/sTKfE9/XkdU+dfsPoQMSrpnQQsEhVFwOIyJPAGUCjKanqG4H004GvxenahtE2dlR404nUfLz5bFradG2ern1cTWf8l7zxjHTm032ArctjGAkmXqbUH1gR2C8HJreS/hLgxThd2zCiNNQ7k4nUehofC5uGVmfmBsb0fDFqPMUjIK9byuQbRmcnXqbUXG+tNnMMEfkaMAk4soXzlwOXAwwcODBO8owOR82OaB/PhgWwYX400i24FHZBL2c4Y0/3xjPSNbf1GGi1HsMIIfEypXJgQGC/DFgVm0hEjgNuAo5U1erY8wCq+iDwIMCkSZOaNTajk6AK29fF1Hh8rWdLoGIuGW5m6pKRMOL4puZjC8MZRloRL1P6EBghIkOAlcB5wPnBBCKyP/B74ERVXbfrWxidlkiT2/r50RpPxICqtkTTZRe4vp7IuJ6I+fQcClm5KZNvGEb8iIspqWqdiFwFvIwLCf+zqs4WkVuAGao6FbgD6Ao8JW5sxnJVPT0e1zfShNoqH9k2H9YviD5XLGo6qLRrb2c2+5ztjWcElIyCbv1sXI9hdHDiNk5JVV8AXog5dnNg+7h4XcsIOVVbXG1n/XxYP8/VeNbPdzNXN47tETeotHQUDD/WPZeMcgZks1YbRqfFZnQw2oeqW4l0/Tzf7LYg+hxcKC4zx/Xt9N0XJpzjaj6lo9wxWyTOMIwYzJSM1lGFrat8U9v8qAmtnwc7N0XT5RS6Ws7Qo6F0pKv1lI6CHoMg035mhmG0DbtbGI6GBhfRFms86+dDzbZouvwiKB0DY8+E0tFRA7L+HsMw4oCZUmejMdJtnn8siPb7BBeMiwQb7Hueq/GUjnaPghIzH8MwEoaZUkelvs7N57Z+HqybB+vn+j6fhU0j3br1d6ZzwEVR8ykZaeN7DMNICWZK6U5DvVtCYf1cbz7+EWs+PQa6ZrZhR0drPSUjbUodwzBChZlSuhBpdls3N2BAPtotaD7dB0Kv0d58xrjtklG2aJxhGGmBmVLYCPb5rJsbbX7bxVa1locAACAASURBVHwGuNrOsKOc+USCDnILU6XcMAxjrzFTShWqsKU8UPOZC+vmuMCDup3RdN0HuL6eoUdCrzHegMx8DMPomJgpJZrIpKLr5vhazxxvQPOahloX9nW1nUkXO/PpNcb6fAzD6HSYKcWTyo0xNR//CK7jk98Teo9zoda9xkCvsa7fJ78odboNwzBCgplSe6jZ4Wo9a+dEm93WzYXta6Jpcrs50xlzmjceX/spKLVxPoZhGC1gptQa9bVQ8Tmsm+1MZ+0cZ0CbltK4hmFWfmBS0dFRA7IZDgzDMPYYMyXwQQcroqYTqflsWBBdxVQyoxOL7nd+tOmtaLCtYGoYhhEnOp8p7agIGM+caBNcMOig+wBnOsOPc8bTeywUj4DsvNTpNgzD6AR0XFOqrXIzW6+dA2s/8wY0G7avjabJL4JePuig99ho01te99TpNgzD6MSkvympwubl3nQ+8yY0261mqvUuTWaun+Xg2Kj59B7nJh21fh/DMIzQkF6mtHNztMazdna0+S3Y9NZjkDOcsad78xkPPYfamj6GYRhpQLjv1Ds3wb9/Gq39bC2Pnsvr7gxn3/OcCfUe55rebKYDwzCMtCXcprRpKbx3r5vZYNAUbzzegCzk2jAMo8MRblMqHQ03fgxZOalWYhiGYSSBjFQLaJXsfDMkwzCMTkS4TckwDMPoVJgpGYZhGKHBTMkwDMMIDWZKhmEYRmgId/SdYRhGGqGq1DXUUdtQS53WUd9QT11DXfSh0e16rY+mjTkWOR7cb+49mnuf+shMNmmKmZJhGElHVanX+l1vwg31u9x0d7kJ+zSt3sybe59m3qvF4wFDiRhMk+u38JoGbUhqPmZlZJGdkU2mZJKVkdX4SGfSW71hdAJUdddSt8bcgGNukm0tZbd6vBVzqG+op1Zr23SjbslMkkmWZJGZEbhxS1aTm3hkvzGNuJt9fla+OxY4n52R3eprgvtN0mdkNTGP5sxkF12B10TeJ7KfKZlICxMICOk7sYCZktGh2N0NPHhzDN7AY0vbrR3f0xt4a2awuyacyGuTSeMNsI038qyMLPKy8twNWAI34MDNvMWbcDPvG3sTbvV4QGPQLGJ1t3TzNsKHmVInp6UbdGs36dibfrDEHHvT3+Mbvn+/1triW6wFpPgGHltqbulmnJ+V32rJuKVjzZXSg9cMGkJbbu57Wvo2jGQQN1MSkROB3wKZwB9V9faY87nAo8ABQAVwrqoujdf1k0GwE7Olm+nuSsGxbdZNzrWxdN2WtLu7cUeafTSyrHsSaEsJPPYmmZeVt8fNHa2V6BubYGJu4JFrxt7AW7q5Z0qm3cANIwHExZREJBO4DzgeKAc+FJGpqjonkOwSYJOqDheR84BfAOe29r7barbxytJXmi0p73FzSgvGEVtab80kkhnVkiEZzd5wW2uiyMnIoUtWl11Kw601ecSW9HdXym7LTb+lJhm7gRuGsTviVVM6CFikqosBRORJ4AwgaEpnAD/1238H7hURUdUWi+rLty3nuv9c1yYBu2s+ae5mGenEbK7UHEzbluaPNt+sI6+T7Kam4tNHOkozxIaQGYbR+YiXKfUHVgT2y4HJLaVR1ToR2QIUAxtaetNh3Yfx9OlPN9teHnszt1K4YRhG+hMvU2rOEWJrQG1Jg4hcDlwOMHDgQEYWjdx7dYZhGEZaEK82onJgQGC/DFjVUhoRyQK6Axtj30hVH1TVSao6qbS0NE7yDMMwjHQgXqb0ITBCRIaISA5wHjA1Js1U4EK/fRbwemv9SYZhGEbnIy7Nd76P6CrgZVxI+J9VdbaI3ALMUNWpwJ+Ax0RkEa6GdF48rm0YhmF0HOI2TklVXwBeiDl2c2C7Cjg7XtczDMMwOh4Wd2wYhmGEBjMlwzAMIzSYKRmGYRihQcIcACci24D5qdbRBkpoZRBwiDCd8cV0xpd00JkOGgFGqWphqkW0h7DPEj5fVSelWsTuEJEZpjN+mM74YjrjRzpoBKcz1RraizXfGYZhGKHBTMkwDMMIDWE3pQdTLaCNmM74Yjrji+mMH+mgEdJH5y6EOtDBMAzD6FyEvaZkGIZhdCLMlAzDMIzQsFemJBLe5VHDrC1C2DWGXR+kh0YwnfEmzDrDrC0daFfmicgwEempqg1+PzO+stpPmLVFCLvGsOuD9NAIpjPehFlnmLWlE3sc6OBLAdOAncDDqvpwAnS1izBrixB2jWHXB+mhEUxnvAmzzjBrSzfaU1P6Om4F2Z8AU0TkDREpEk985XUobRHCrjHs+iA9NILpjDdh1hlmbemFqrb5AfQE3gO+6PfHA/8AugbSZO7Je8brEWZt6aIx7PrSRaPp7Fw6w6wtHR97WlO6Glilqs+ISBZwGLAQOFVEbhSRIlWth5S0p4ZZW7poDLu+dNFoOjuXzjBrSzvabEoiUopbwvzn/tDBuOrqMlV9EqgE3hSRawFUtT5Z1db2aPOvE/+c8B9K2DW2V1/g9X0SqS9dNHYynSWdXefe/q+NXWmzKanqeuBwVf1IRHoARwMKbBeR3wHvAEcCI0TkXRE5TX29VUTKROR8EXlMRE6L94dojzb/ukiUx0ki8kEitKWLxvbqAxCRgcA3RGS6iJySCH3porET6cwBzhSRGSJycmfVuTf/awmEjnudBrS9TwnICGyXAjcCZ/n904C3gd/h2lcnAH8DvogzvquBp4HLgA+AW4DseLVBtldb4DWDgd8DHwFjE9FOGnaNe6MP6ALkAqcAM4CLwpaHydLYCXR+KeY9TgfeB77dGXW2Q9tTkfOB113stf0ByE1UPqbLY2+/kOyY/f8BPgG+CwiQ549fD9zktwcDrwBdEvrBWtHm93MD5wYA3wFui3lNQjsn91Zj8A+RIn05sXkFHAM8nOi8i5PG4A1FQqwzaZ3kbf1NAl0jmoF9cUaQ5f/3WemgMxXafP5EhuJc7o/9C3gI+BFmSu0ePCsAqlrr9zP9/m9xVdXewBSgwDdBHAp8S0RuBH4NrFPVyvZcOw7aikXkRFWt9uczcO3A44BnAu8zFnhBRL4VVo3+XL6IHJIifTWR12i0Hf96oFJdn2KJiBwkIhfEU1+cNG5T1YZIE4r6u0QIdVb5vBzqm8AvSbHOav+SAcAbInIrcA+wVVXrgCOA1xPxv4mTzp1eZ76IFIvIF5KorRdwIDBRRG4HxuBqTq8Cy4FXVLVaRPqJyMmJ+q7DTlxnCReRzMAfChH5GZCJKwmsBR7D/TD+qapVPk2WqtaJyKnAelV9P26CWtEWOL4PbozBGlW9K3B8MC608zKgBrhCVRO6DHI7NN6GW555JFAMXKmq7yZDn4hI5EYuIqOA44CDcDXhY/yN9E1gNrAfUAFcpqprE6WvjRoPBIYBRwXSlQAnAl8Bvq+qsxOpcQ90DgWO9nlZDjyJu5FlA1ep6oJk6gwcE1VVEZkC3A78UlX/5c/1A8biavUVuOay7SHR+QtVfcGfewzYisvPHl7ntGRoE5HvAdcA3wTeBW4G5qrqH8UFuTyLa87bHygHvqmqWxOhLYzEdTl0jUbcCc6MuuKWNJ8GICK1uBJqVaBUUedf/hTwNdyXEXcC2jK9CUZ+KPvjfpT3eI2ijqXAUuB5EXkNV8pJqCm1VaPXeQmu3+EwVd0hIl8BjhCRz1R1SwL1ZaibRqW/iJwETAYmAm/impo+8Ol+iCuVftvrfQWXhwk1pTZofASY6dMNxNU+L8X9+Y8EuidS3x7ofBiY5dONxoUcXw8gIn8GRgEJN6WgThEZABSp6ix/bpqINABrAulXAauAf4vIO7hCU8JNqQ06FVjtay9XAoNV9XAAEfk6ME7cUue1CdLWWPBQ1TtEZBHwS6ABN8bpcREp9tpmqOo1XtvbuDw0U2ovPuMVaPA/hIOAB8VFviwFVvqkkabDel+Vna6qTwdKN41fYpy1RUzwJRGp8Zr+rqorAj/q7EAV/HhgPa50mnB2p9FrKgF+jCtlfVFE/k9VnxCRHokypIC+Br85HBewshQ4QV0UEl7fJOBwXKcvIjIMWIErqCSc3WkUkVwR2R/X1v8hcCeu2elxVX0vGRrbotNrzQaOB/JE5G5cf2wW0C8FOnsAD4jIa6p6s4hcDGwDPo9oDfxvzge2AJtDoPMSr3MhroB3FTBLRL4JPKSqj4lI10QYUkBbpCac4XefEZGtuFDyKbhWmYG47/WXPu3BwGpgU6J0hZFEz2b7/4AcEZmJu0G9DcwDV3rwJYhSXOngfP+a/f35hLTxB7gQV/O5FBjtr9ngn2tFZICIXISrnbyB+3FEflSISDcRmZBsjZ5bcaXPe3EhqH/0zaCbvbbI2KaEaVTVN3HRRB/iapMXBk5/Hdcc8bHPr15AIW5esMa2dnF9TlMSoW83GgcDvwD6qOp9uNrbV4neDDICeZhQja3pFJECr2kkLpKrP/B94Algqk+TlUSdnwJfAvqKyHRcnv0x8rsDMkVkrIj8ANdc9gcg0kwf+d+kQuf5wB98M+IluP/yD3A10zv8a7YnQ6eqNvhCdzdcM+3fvdYFwCRgtqouEBciPsRrzfeaIv+bfiJyXLy1hQZNXARKMLJpBDDCb/cD7gIK/f7vcKWY84DXcAZwUuQ9SECUTIy2I3E/zEygL3AOLhLmNeA3wOktvMdXgT8DByQh/47E3ZzygW44cy/z5/p6HaXJ1Bij7zDgB367J65EOiDwff8S+H/NvMeNuGWbByYhDw/D9RdF9g/ANZu84R/Xx74mGRpb0HmD3x6Ha1noHjh3n9+WFOssAwr89hhcFNlU//gJMLmF90iFzm5+uxj4NPJfAQ7BGWdeivJzEHCQ3+6La00o8fuTgF/h+rJjX/d74AEgP1HaUvmIe/NdBHXNYJF+o4Xgmk2Au4EKVd0mLmrsQmAxrkR1Da4Pqs433zVOAa/NBADESdt/gP/469yDK7X8UFW/EXxNpK/Mv3YIrmT7CTA3mRqBnSLyLlCE6wfphQskiJTwMpKhMUbfO7hBguBqQ+/gmmXzgbNwzRLXeg05qlrja3CjcSXFtfHW15pGn0cfAYf47/xSYJZ/WaaIRF6bcI2t6cT1zc4i2p9QhCv991DVzREtKdJZ7q9TiCu8HQ9cqKqPBV/jax6Sap2e7cBbuH4ccPlZjLu5V6UgP5cBy/xuAzAd2CYikcJxAy6EPfi/ORQXAHMdLgArIdpSScJMCZptghsKLFDVG/3+s7gIlGJcfP5saDSAZ0RklqreHMnwON9UI228wb6rc3A3z/NEpF5VfxWTPpLuJJyJvqw+tD0RP4oWNILrWP6riHyIqzn9XX1Um0bb1ROusTl9qrpTRBbjpvFfgLup3qmqq/35SOjzBbiBwG+oD99NVh76m1YmbqDqFHznsj8X7FdIisZWdH7mv+NZIrIA931OVd9cFtCSdJ2B/W3igjS+C1wtIv1xUW6NeR1InjKd/li1iKzETfszB6gGnlXVTTFakqYzoG2tiMwDFuH+N+8CD6hqhT8f+d98F3hMVWf5AsG2jmRIEOeQ8DZdMFoa6YZr0rlKRK7HmdL/+jR9gB3AT3FV7DtV9e8J1hWJhIqEg38Pd0OfKiIH4SLJPhWRfXHhrltxfQDjcW3+31HV5c0YSKL0DsT1M/xLVZd6XaKqn4RBo785jVLV1/3+qcASVZ0tIkfgBgq+jSsNDsQZxPXAOkhKn2KklH+tqt7i908HFnszCIVGr6sM1/8wVVU3ehNYEQadzfxvvgP8VVU/9Pm5SFXnpFpnjOYyXF/sc77GGab87IPr6/zE718GvKWq831+/hzXdLcPLlJ0PS6kPFJQSe4NPQEk3ZSaFSFyIi4i5ixcbeRZYCOu1tID90VEbiCf+ddEovSGAajq53HS0vgn8/uTva4NuL6R7+M6SA/EBXJsxf1oVgRek9DqdDMaj8KFh4dCYzP6zgaOBV7HTel/D25QY6nfXg18rKobk6GvBc3n4GZQCJXGFvLyOFyfZ5h1no77TYZdZ2jyM9D0Hrm3fRM4Chcd+hlu4Hwu7n/+F1y3xxJV3ZFobUlFU9SZxa4di4cDxYH9b+Kq0D/EBSF8ETf4MnK+D+7H9A9ciSYR88Hl4Do7v4xrJjsGF/H2Q+DfMWmPwYVpdw8cS/j0MLhQ9QdxfWGh0+jz8P9wg1XB9d/cB1wBvBiT9mxc1Fa3JOdh6DWmqc6j00Rn6PLTa3saH6QE/C9wPy6S8G8xab+FM9Ck52EiHokOCW8R9TkX6JB8W1UrxIe4qurvcdFGPYE5wBZV/YOIZIkbKPoArlqdh2vWmJMAjZGZHJ5WN6I6H9fx+TugSkQO8J8hV10zVSXwHxH5H//6SF9YwvJZXR/IFar6j/ZqjHwHCdJXA5yvqm/6fNiJGyD9tJPlaroikq+qT+GaI/4jIt+J6Au+n29y63Qa01TnG2miM3T56bWdo2728Uxcje3/cH1NBeKG0iBufNXvcGPc3gxqi/yvxQ1xSPTwlfiRalds6UHA6XHTlhyBa9/9I66JqhDXproZGJ2s0gHRSWZvBc7z2ycAj+ImfOyJM4R3gJNjXjsQ15b9hTBqxDVbfA03HdQpif5ucSW/E/z2V3HRThNi9J0SeF0XXCf0f4ETk/H7C7NG09n5dBKt3R3s968BXojR9m6Mtv1wrSkfRj5TmB8pF7CbL0BoOuZgAC6q61V/7gng9/7cLjNm+zQJmQ0Y19z4P4H9M/yP4T5cP9j+uBLXCf6HdAiuXfh/cct3vAT0SnD+tUXjU7hp/TNxfXh/x81ePAPXgZqwGZ+9vgsC+6fH6JuIG2d1eOQ7xo3niIyH6ZeE3+DeauyfaI3pkpemM27aDqNpYfK0GG0H4IbejPPnu/rn44GZwLHJyMN2f75UC2jnl3INbqBZNdGBha0u40Dy2qmvw40NCprBsbjOym8Gjt2Mi7JJRf410Uh0mv8biA6CHYEzzrwkaZLA9ndxoeRX+/2swLmJuBrgxSnIt3Zr9DetZP0G9zovSVBhLp75mQ7fewq0fYqbrLcxj2hasP8hgb75MD5S1qfUHgL9TXfj5oP6HIgMctVAujEicoMPp4zMITVd3ODdRGmLTFN/J66Jro+IHCEiPXGBEgfgVo+9xKe7RVXXJLI/pw0a+/nowkJxK2IeDFwlIjfgZt1YpX4290SjqhrQdxduFosCETlQ/aS94uaB+wLuBv9G4HP1E5FTxc3FFiaNkXD4XFzJ+00RuTyRGvdSZ2bwPcKq05MjIqUSx6Un4qgz+NvcR0SuE5FLk6TtCEBFZKRGp02LhOzvj4vKjWgbISIXR+6TYSEUIeF7ig83/bOqlvj9HPWDy8TN+HsWbqqbYbjq6om4QXK/TIK22Gnqx+FKT/8CXsT1if1K/fidVNCMxlv95vM4s38IV/1/RgPr/KRKX+D4ibhBwW+q6jP+2ABcBOa7uMl/F+MCP7alWOMbqvrPSFpciPEEXBPpKuAaDYTyplBnMC8jIcmjgTNxzT2XqJsxPzQ6/fFHcdOTjcVFnV6hqh+EQGfwey/E9TE9hhueUe91lse+PlHaRCQPZ5ZH4yb+/UBVbxU39vJ+4N+4AtOnuN9k0v/vsSR0RodEoW5A6/4APqrkdHETL76DG/R6raq+5s+/jmvmu9PvJzSOXwNRL760Ocxv/slffwsu4KG5mRoaae1cHDUK7jfQFfhU/VpW4mZ336RuWpPWNCYkL2PzUKJjSybjIqDe9tcfjpv8dbqqfscfewfX4ZtQU2qDxre8HvF5tMY/XhG3HEFP3ADxhNLWvPSM9v+ni3AzC/TCdeAnnD3IzwJc6PZAVT3KH7sIGC8iH2sCZ/puo85gfo4CylX1J17nP3Fz8SXElLTp8h1H4vrBBuJmzHkCuEdVF4vIfsC5wBO+1QT/m+yBHyCcStKq+S6IuqUmCoGrcWHhS3EdgHOIzmWXiSs93xP5MWk0TDthyyiox+/uwC12FhmIuwo3pb8/1OKsxAeLm/4/kRobfMlI8NV634y3kOgaOY2/EXGrnwabS871tdZE6VMR6QK8482mCHhKVTeIm0V5Im7C1994fYfgJjBNynIJu9G4MWLo4pudvcYLcN9/QpcYaaPOv/u8LPTf4w+AWtxEoKtw42HiPtSiHTqfUj/dDm52kiuBdSJyubjZ8R/2WhNqSG3Q+Xd1w1oyxC0vczquifw3voksssZcIrVFBgaPxEXS1qnqMar6B29Ihbia03acUSEiZwIbVTXlhgRpbEqeelyE282qugi3pHCGRhcOvAM3k8EzItIVeNXfFBpLPJLAMUSeaUA3cXOY/T/cGKL3vIYGX6rZD1cSvFFEZorICbjBsEckWFuEW3Ft5B/hgh3+A8z3GutFJE/culIvAVeKyPvixoodRNMlNeKOunn7zgGW4JYdOMafqscVQmap6ufi+m0iU/3nJVJTWzUGCiZZ4voWfgjchpvlubq590qizotxTTrgpqz5MVCrqk/j7guH4JqaEzrOrg06g985uEH1K3AGegh+uRFNwuq2sTSjM5ifP8KtizYZv9Ixbs2spKzXpap/wBn4dv9/PdOf6ooryH2iqqv8fXEc8K6vhaYeDUG0RTweuD9SL9wiaA/iQq+rgPH+fE9cm+r/4fp2Dgy8NhmRRuOIjs7ujftTFeLCSB8GfuTPnQL8FXiO6PIUiQzLDkbmjCK6xMhAXIk5B3fzvxe43J87B1cYuJfoMgAJiSyL0Xcc8GO/3R9XCOnh9w/CNdFe3sp7JUPj8cCNfnssLiLqOVyY8I9pYUmHwOtzkvRdHxf5zfn9ybgC1HvAy8C5/niz/41E5WVrOnH9cp/iVpUF1xfye9zUOy3pTGZ+3hzYnk40qvUbwB3JzM8YbYfgBgmDq0HNwy97gZsp5y5ixlS29n6JfiTlIsl++JvoO8C9fr8UN+7mFty0PMfhaiaPRW5qwYz3N7zucdKyy1gpf917/PaZBKYDwgVlfEJgjIQ/nsibQBONuJrGP4G7/f6lwJ+IDiy8wt/Ajk+GxhbysB9uTFUmbvzaHbiF+yImleOfc3Gz0/eMaMQNKdhlDZ0452EBLlqsPnJDiEnfGCLuX9sncrP1x76P68+Ja4GphbwMhjU/hgt2iazplBXQmZS8bEVnPq5zPvIdn44rHEXWSwpLfh4LvOK3uwDf9r/PiEllJSM/W9A2HjcDDrjAm7/6/3NBjLbuuEG3xX5/BG78Yk688zD2kZaBDi0R6eRT1XfEzfabDaCq68WFhf8MV4K5U1WPEJGrcEEST6hrjx4kLprrDmBfESlT1Q17o0n9NxrQONRvXuef9wfe9Od64gx0IfC478g/Wl17cOOURdp0OYC9JlYjrrlhtqreFND4ibqmvCG4P9rbwGs+n8er6v2J0hjRFwy6UNf0sBAXbfc5Lg8f1OiyDpEooudwN9mDROS7XvvV6oYVxI1Yjaq6w/e/XQtc639LvwykD+bPA7gIsiki8i2cYd2uCYgWbSEv63wf6wSgBBc8MjtyLvDypORlKzp3ishq4G2JLj3xjLrptdBo0E1K8xNXGDlORD7D1ebLgf/T6DIYkTxNaH42879G3Uzo80RkOa7G9IKqPhA4X+fvQ//GBbscICI34/rMTlbVB+OhbXfCO9yDXVcPDU5ZNBR4HGdOo/yxbrj24H8DJ+PWM/lJ5PeWAH2RUrz4636GqyHdhms6OcWfL8A1Rb4LnNnS50tQHkZKTBm4cVb/xTUt/gV4kuiKmQVEpzb5YlBj4D2yE6izH7BvYP8yok2Qt+KWHwHX33S//xynJkFXsPlkGPBbos23pwNj/PYtuD68EtxKpLfjZoE+P9Eam9HcA7gysH95GPKyGZ2DcMYZGTh/EtHZC1KanzStLQ/EtYREWmAuDkt+4mqSAwL7N+NaHApwXR8P++PD/X/+BeAIfyxh3QmqHbT5rqUfS8yN4lhcM14xzgiuAgbjxhPMI9CslwRtJ/kfwmvA/f7YJFxUWS/cctPP4aau3yf2c/nn3ARrPAHXPPYK8D1/7DTc7MQRjVNxy46Mj9H2PM60xsRZU0bMda70f6CJuJLdcmCIP9cdV5t6OOY9Ej1LemwB6Ys4Ez8LVzJeCQwLnH8JNyYnaRpb0P1tXFNeaPKyhfw8F1czClV+NqPz22H5bTaj7Ub/vy3GFdpn4JeBx/WLTgd+lqw8TOgXE8ZHM1/IGFxJ6td+/w3gW3474QEQMVpOidzQ/f51uEkUf4CreZyGW9r8+ECaPJyRvgr8JPbzJUDjkcDQwP61/kcc0Xg6rmnvsED6NcD/4GZgvgPXdh7vtv1c3CDafXEFkEOBfwTOD8NNDxOpoVwIFAbOJ2UyX9zy1ofjaudDgCcD53NwzTmRGtXZydaYLnkZ0Gn5uXfaCnGD5Yf7/S/hQtsj+Xc0buD/IJ/2ikRrS/gXkg4PXEnlV/7mWZ6C68d2RgabALrgakzTgLNj0h2Daxa6H9dk8acUavw1bqLZM2POLQEu9ds9gG8kUGOwo7sEtx7Xt3Bm/jJwlz+fgev4bpzDLPAeiTb1YFNyV1xAzrW46Mx/48bk7E5jMgw0HnmZNJ2Wn3HTNhgX4fgN3BjQF3ArVge1/TeR2hL6RaTDg0DIKK5jbz6BvpEUawv+4cbhms7G+MfVuFmBT8TNWfc0vm8l0TfW3Wj8C76JERfuPsP/sI+NeV0ywvD3wTWLPoLrJ+zq/1i/INqp/0ecoX8hVh+uv2q/BGschxu/9kdcpF4ksuxnuJJ1UOMJMa8djSv9J3QplDjk5YG48UXJWHrC8nPvtU3w2h7HDZ/Jx/VB3e11lwB/aCEPD8VFxbYaYt7q9ROd+enywHX0PeO3k3ZTb4MuidWDm+l3MdFp83+Jiy4KjcaY86fg+ie6JVFTsP/wMPy6Un7/LFwf1z24Zp+JuGbSyPiiAlzt9GFck+MrJGApAnbt4wzOIn92MxqnA7f48z1wsy78xB//KwnqB21HXn4QynfBpQAABQhJREFUyMveuIUlb/B5+RCuGSgRAUSWn/HVNomA8eECnmK1TcPPSu7TjMXVVOvx96c91pCITE/XB9GY/KT2Je3pj8Xvfw9Xs3se1+ab8uWPiQYelNJ0jEgRrhlvSAg0BpsWv4drRoks4REZlPlnAk2huELABSQ46qiNGiMDqo/A90vg+uj+CkwMUV5Gxt6cGdBZiOtbServwPIz4dryY9KeBfwnsL9Hhfx0n2YoLgQmWKzwz5paRbui0ennM/z+HcB5uFkYRgGDkz0dTCwaHXszBHhdRCLLilyOM86ETpLaFlSbTPV/B24esENE5HpV3eSnUzoDWCsiZ/l0P8cNOKxr8Y0Tr/FgEfmJqpb7sXSDgEkiciWutC24UmrSaEHnoSJytbq5/0bgBl0e4scLnYNbcnxoi2+aPJ2Wn3uv7SARuUhVd0bSiUhvXL/T7YGXZ4pIfxE5uS3XSsulK4zG2ZK/jgt2+BmwXBO8XMOeICIH4sZdZeCizm7ATZuflEkzd4cviEjA7LPUDRycjetk/i2uXf0l/ycMg8YCdYNyX8Y1KX6C64NYhWv/fzZ4g0ihzkxcH8Q9uE78mbiw7Tm4fpLpmpolUSw/46MtQ3ddIiNT3eD6W4GRqnpu4Nwvca0lE3Am+g1VXdLSNTrUjA6dCf9n+gxYqKqzJYGznu8pfnT7h8AJIjIKWKeqm1KtK4ivDTcuPeANaSJQoapfARCRnwMn+rxtSHYNOqjR7+4QkS8Do1X1BK/xRmCKqj7p9xO25EkbdeJvTifg+uF+73U9hBsQ+lYytbWi0/Kz/drqvTbx2rJVtdb/18/E9d/hdV4DTMENXl4hIg/jWlLMlDoiqvpOYDtha0TtKb6qH7nZz0+1ntbQptP9LAI+jdSacGvfjE113sZoXIfrbI7Qn8Bqoqlseo7RuRk3XRYi0g035dfwVOiKxfJz74nRdruIVOKaQB9X1XlAZMXlHwOnqeoKn7YcF+XY4iKnZkpGQtA4z8+XJKpwf/Z3ReQDXF/dryExcw62k48AxC1q+TGujyGyUFtYNIIbzH2uiLyPixRtwI2nCxuWn3vPnbhw8S/hxl5F+DnwmqpODxz7CvBVaLkWan1KhhGDuMUChwJvq+qyVOtpDq9xJG69sMWp1tMS4pbd7oeLxgpVE24Qy89262k0bhE5BjcbxW9w4z0fAn6jqtP8+V/j5v47tbVmUTMlw/AEojBD+6dIB41gOuNNmHXGagv0Md0FvK6qz3vTfwq3nlh5a6ZkzXeG4QnjHz6WwB8/6R3we0KYtQWx/Nx7msnDyNCJZcCd4lb73o4bqFy+u2ZRqykZhmEYCUFEIkvB/6OtzY1mSoZhGEbcia0RtbU2aqZkGIZhhAabZsgwDMMIDWZKhmEYRmgwUzIMwzBCg5mSYRiGERrMlAzDMIzQYKZkGIZhhAYzJcMwDCM0mCkZxh4gIgeKyCwRyRORAhGZLSLjU63LMDoKNnjWMPYQEbkNt5puPlCubrl0wzDigJmSYewhIpIDfIhbf+mQVC8CaBgdCWu+M4w9pyfQFSjE1ZgMw4gTVlMyjD1ERKYCTwJDgL6qelWKJRlGh8HWUzKMPcCvDVOnqo+LSCbwnogco6qvp1qbYXQErKZkGIZhhAbrUzIMwzBCg5mSYRiGERrMlAzDMIzQYKZkGIZhhAYzJcMwDCM0mCkZhmEYocFMyTAMwwgNZkqGYRhGaPj/R/J8uJy01s4AAAAASUVORK5CYII=\n" - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "import scipy.special as sc \n", - "import pandas as pd\n", - "\n", - "def f(n:int, d:int, p:float):\n", - " a = 0\n", - " for i in range(d, n):\n", - " a += sc.comb(n, i) * (p ** (n - i)) * ((1 - p) ** i) \n", - " return a \n", - "\n", - "n = [i for i in range(8, 42, 2)]\n", - "k = [i for i in range(4, 21)]\n", - "d = [4, 4, 4, 4, 5, 6, 6, 7, 8, 7, 8, 8, 8, 8, 8, 9, 10]\n", - "nkd = list(map(lambda x: \"n={},k={},d={}\".format(x[0], x[1], x[2]), zip(n, k, d)))\n", - "\n", - "composed = {'x' : nkd}\n", - "probs = [0.1, 0.01, 0.001]\n", - "\n", - "indexes = ['x']\n", - "indexes.extend('p={}'.format(p) for p in probs)\n", - "\n", - "for p in probs:\n", - " ans = []\n", - " for item in range(len(n)):\n", - " res = f(n[item], d[item], p)\n", - " ans.append(res)\n", - " composed['p={}'.format(p)] = ans\n", - " print(ans)\n", - " \n", - "df = pd.DataFrame(composed)\n", - "df = df[indexes]\n", - "df.set_index('x', inplace=True)\n", - "df.plot(rot=-30)" + "dataset = il.CustomDataset(train_tensor)\n", + "l = torch.utils.data.DataLoader(dataset, batch_size=8, shuffle=True, num_workers=2)" ], "metadata": { "collapsed": false, @@ -496,18 +63,7 @@ "cell_type": "code", "execution_count": null, "outputs": [], - "source": [ - "import scipy.integrate as integrate\n", - "import pandas as pd\n", - "\n", - "n = [i for i in range(8, 42, 2)]\n", - "k = [i for i in range(4, 21)]\n", - "d = [4, 4, 4, 4, 5, 6, 6, 7, 8, 7, 8, 8, 8, 8, 8, 9, 10]\n", - "nkd = list(map(lambda x: \"n={},k={},d={}\".format(x[0], x[1], x[2]), zip(n, k, d)))\n", - "\n", - "composed = {'x' : nkd}\n", - "probs = [0.1, 0.01, 0.001]" - ], + "source": [], "metadata": { "collapsed": false, "pycharm": { diff --git a/properties.py b/properties.py new file mode 100644 index 0000000..a873273 --- /dev/null +++ b/properties.py @@ -0,0 +1,23 @@ +data_inputs_path = "/home/nikita/PycharmProjects/ISIC2018_Task1-2_Training_Input" +data_labels_path = "/home/nikita/PycharmProjects/ISIC2018_Task2_Training_GroundTruth_v3" + +tensor_train_path = "/home/nikita/PycharmProjects/tensors/test" +tensor_validate_path = "/home/nikita/PycharmProjects/tensors/validate" + +# for image_loader +classes = [ + 'pigment_network', + 'streaks', + 'globules', + 'milia_like_cyst', + 'negative_network' +] +field_id = 'id' +field_train = 'train' +image_size = 224 +num_workers = 5 + + +train_size = 200 +test_size = 100 +batch_size = 8 diff --git a/requirements.txt b/requirements.txt index e69de29..a035fdc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,4 @@ +scikit-learn +PIL +torchvision +torch diff --git a/utilits.py b/utilits.py index 872dfd2..12eb513 100644 --- a/utilits.py +++ b/utilits.py @@ -2,7 +2,7 @@ import torch from torchvision import datasets, transforms - +import torch.utils.data.dataloader import os From 05647f537d0fc5a86f4c640c59b2d7ff0385b718 Mon Sep 17 00:00:00 2001 From: glcanvas Date: Wed, 6 Nov 2019 21:47:59 +0300 Subject: [PATCH 04/10] isic wrong classifier --- image_loader.py | 34 +++++- model_settings.py | 0 notebooks/init.ipynb | 270 ++++++++++++++++++++++++++++++++++++++++--- properties.py | 2 +- 4 files changed, 281 insertions(+), 25 deletions(-) create mode 100644 model_settings.py diff --git a/image_loader.py b/image_loader.py index ccb9176..608c20a 100644 --- a/image_loader.py +++ b/image_loader.py @@ -12,7 +12,7 @@ attribute = '_attribute_' composite = transforms.Compose([ - transforms.Resize(image_size), + transforms.Scale((image_size, image_size)), transforms.ToTensor(), ]) @@ -115,16 +115,38 @@ def remove_empty_mask(cell: dict): return cell +def apply_resize(cell: dict): + scale = transforms.Compose([ + transforms.ToPILImage(), + transforms.Scale((image_size, image_size)), + transforms.ToTensor() + ]) + for c in classes: + cell[c] = scale(cell[c]) + cell[field_train] = scale(cell[field_train]) + return cell + + class CustomDataset(torch.utils.data.dataset.Dataset): + """ + data -- list with dict of result load_tensors + """ def __init__(self, data: list): self.data = data + self.inputs = [i[field_train] for i in data] + self.labels = [self.__item_to_tensor(i) for i in data] def __len__(self): - return len(self.data) + return len(self.inputs) def __getitem__(self, item): - return self.data[item] - - def __iter__(self): - return iter(self.data) + return self.inputs[item], self.labels[item] + + @staticmethod + def __item_to_tensor(d: dict) -> torch.Tensor: + t = torch.tensor([0 for _ in classes]) + for idx, cl in enumerate(classes): + if cl in d: + t[idx] = 1 + return t diff --git a/model_settings.py b/model_settings.py new file mode 100644 index 0000000..e69de29 diff --git a/notebooks/init.ipynb b/notebooks/init.ipynb index 4fb4ead..33884b2 100644 --- a/notebooks/init.ipynb +++ b/notebooks/init.ipynb @@ -3,35 +3,66 @@ { "cell_type": "code", "execution_count": 1, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "text": [ + "/home/nikita/anaconda3/envs/ml-diplom/lib/python3.7/site-packages/torchvision/transforms/transforms.py:210: UserWarning: The use of the transforms.Scale transform is deprecated, please use transforms.Resize instead.\n warnings.warn(\"The use of the transforms.Scale transform is deprecated, \" +\n" + ], + "output_type": "stream" + }, + { + "name": "stdout", + "text": [ + "True\nGeForce GTX 1060 with Max-Q Design\n" + ], + "output_type": "stream" + } + ], "source": [ - "from sklearn.utils import shuffle\n", - "from torchvision import datasets\n", + "from torchvision import models\n", "import torch.utils.data.dataset\n", - "import os\n", - "from torchvision import transforms\n", - "from PIL import Image\n", "import torch\n", - "from multiprocessing.pool import ThreadPool\n", - "\n", + "import torch.nn as nn\n", + "from torch.optim.optimizer import Optimizer\n", + "import torch.optim as optimize\n", "import image_loader as il\n", "import properties as pr\n", + "import copy\n", + "from torch.multiprocessing import Pool, Process, set_start_method\n", + "\n", + "#torch.multiprocessing.set_start_method('spawn', force=True)\n", + "device = 'cpu'\n", + "#torch.set_default_tensor_type('torch.cuda.FloatTensor')\n", + " \n", + "print(torch.cuda.is_available())\n", + "print( torch.cuda.get_device_name(0))\n", "\"\"\"\n", - "r = il.map_inputs_labels(il.data_inputs_path, il.data_labels_path)\n", + "r = il.map_inputs_labels(pr.data_inputs_path, pr.data_labels_path)\n", "\n", - "train, validate = il.split_set(r, il.train_size, il.test_size)\n", + "train, validate = il.split_set(r, pr.train_size, pr.test_size)\n", "\n", "il.prepare_data(train, validate)\n", "\n", "for i in train:\n", - " il.save_data(il.tensor_train_path, i)\n", + " il.save_data(pr.tensor_train_path, i)\n", "for j in validate:\n", - " il.save_data(il.tensor_validate_path, j)\n", + " il.save_data(pr.tensor_validate_path, j)\n", "\"\"\"\n", "\n", "train_tensor = il.load_tensors(pr.tensor_train_path)\n", "validate_tensor = il.load_tensors(pr.tensor_validate_path)\n", "\n", + "\"\"\"\n", + "train_tensor = list(map(il.apply_resize, train_tensor))\n", + "validate_tensor = list(map(il.apply_resize, validate_tensor))\n", + "\n", + "for i in train_tensor:\n", + " il.save_data(pr.tensor_train_path, i)\n", + "for j in validate_tensor:\n", + " il.save_data(pr.tensor_validate_path, j)\n", + "\"\"\"\n", + "\n", "train_tensor = list(map(il.remove_empty_mask, train_tensor))\n", "validate_tensor = list(map(il.remove_empty_mask, validate_tensor))" ], @@ -45,11 +76,108 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 8, "outputs": [], "source": [ - "dataset = il.CustomDataset(train_tensor)\n", - "l = torch.utils.data.DataLoader(dataset, batch_size=8, shuffle=True, num_workers=2)" + "def get_model_parameters_to_update(m:models.AlexNet) -> list:\n", + " params = []\n", + " for name, param in m.named_parameters():\n", + " # maybe here better check require gradient back propagation\n", + " # and then update\n", + " # print(\"name={}\".format(name))\n", + " params.append(param)\n", + " return params\n", + "\n", + "def set_model_last_layer(m:models.AlexNet) -> models.AlexNet:\n", + " num_features = m.classifier[6].in_features\n", + " m.classifier[6] = nn.Linear(num_features, pr.labels_number) # here 2 for each label \n", + " return m\n", + "\n", + "def train_model_single_epoch(m:models.AlexNet, data_loader:torch.utils.data.DataLoader, criterion:nn.CrossEntropyLoss, \n", + " optimizer:Optimizer) -> models.AlexNet:\n", + " m.train()\n", + " for inputs, labels in data_loader:\n", + " inputs = inputs.to(device)\n", + " labels = labels.type(torch.FloatTensor).to(device)\n", + " optimizer.zero_grad()\n", + " \n", + " model_result = m(inputs)\n", + " #print(model_result)\n", + " #print(\"-\" * 20)\n", + " #print(labels)\n", + " loss = criterion(model_result, labels)\n", + " print(loss)\n", + " loss.backward()\n", + " optimizer.step()\n", + " return m\n", + "\n", + "def validate_model_single_epoch(m:models.AlexNet, data_loader:torch.utils.data.DataLoader) -> float:\n", + " confusion_matrix = torch\\\n", + " .tensor([[[0 for _ in range(2)] for _ in range(2)] for _ in range(pr.labels_number)])\\\n", + " .to(device)\n", + " \n", + " m.eval()\n", + " for inputs, labels in data_loader:\n", + " inputs = inputs.to(device)\n", + " labels = labels.to(device)\n", + " # trust result\n", + " for t in labels:\n", + " for idx, v in enumerate(t):\n", + " confusion_matrix[idx][v.item()][v.item()] += 1\n", + "\n", + " model_result = m(inputs)\n", + " # all vectors size 5\n", + " for i, t in enumerate(model_result):\n", + " # all classes\n", + " for j, v in enumerate(t):\n", + " dst_0 = abs(v.item())\n", + " dst_1 = abs(1 - v.item())\n", + " trust = labels[i][j].item()\n", + " model_answer = 1 if dst_1 < dst_0 else 0 \n", + " confusion_matrix[j][model_answer][trust] += 1 \n", + " \n", + " print([calculate_f_measure(i) for i in confusion_matrix])\n", + " return sum([calculate_f_measure(i) for i in confusion_matrix]) / pr.labels_number\n", + "\n", + "def calculate_f_measure(conf_matrix:torch.Tensor) -> float:\n", + " precision_w = 0\n", + " recall_w = 0\n", + " sum_all = conf_matrix.sum().item()\n", + " \n", + " for i, t in enumerate(conf_matrix):\n", + " c = 0\n", + " p = 0\n", + " for j, _ in enumerate(t):\n", + " c += conf_matrix[i][j].item()\n", + " p += conf_matrix[j][i].item()\n", + " if p == 0:\n", + " precision_w += 0\n", + " else: \n", + " precision_w += ((conf_matrix[i][i].item() * c) / p) / sum_all\n", + " recall_w += conf_matrix[i][i].item() / sum_all\n", + " return (2 * (precision_w * recall_w)) / (precision_w + recall_w)\n", + " \n", + "\n", + "def train_model(m:models.AlexNet, train_loader:torch.utils.data.DataLoader, validate_loader:torch.utils.data.DataLoader,\n", + " criterion:nn.CrossEntropyLoss, \n", + " optimizer:Optimizer, epochs:int) -> models.AlexNet:\n", + " best_model_state = copy.deepcopy(model.state_dict())\n", + " best_acc = 0.0\n", + " for epoch in range(epochs):\n", + " print(\"Epoch: {}/{}\".format(epoch, epochs))\n", + " print(\"-\" * 10)\n", + " m = train_model_single_epoch(m, train_loader, criterion, optimizer)\n", + " accurency = validate_model_single_epoch(m, validate_loader)\n", + " print(\"acc:{}\".format(accurency))\n", + " if accurency > best_acc:\n", + " best_acc = accurency\n", + " best_model_state = copy.deepcopy(m.state_dict())\n", + " \n", + " best_model = m.load_state_dict(best_model_state)\n", + " print(best_acc)\n", + " return best_model\n", + " \n", + " " ], "metadata": { "collapsed": false, @@ -61,13 +189,119 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "outputs": [], - "source": [], + "source": [ + "### EXECUTE ONLY ONCE!!!!!!!!!!!\n", + "model = models.alexnet(pretrained=True, progress=False)\n", + "model = set_model_last_layer(model)\n", + "\n", + "parameters = get_model_parameters_to_update(model)\n", + "data_train = torch.utils.data.DataLoader(il.CustomDataset(train_tensor), batch_size=4, shuffle=True, num_workers=4)\n", + "data_validate = torch.utils.data.DataLoader(il.CustomDataset(validate_tensor), batch_size=4, shuffle=True, num_workers=4)\n", + "crit = nn.SmoothL1Loss()\n", + "optim = optimize.SGD(parameters, lr=0.001, momentum=0.9)\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% \n", + "is_executing": false + } + } + }, + { + "cell_type": "code", + "execution_count": 10, + "outputs": [ + { + "name": "stdout", + "text": [ + "Epoch: 0/10\n----------\n", + "tensor(0.6407, grad_fn=)\n", + "tensor(0.3992, grad_fn=)\n", + "tensor(0.1331, grad_fn=)\n", + "tensor(0.1047, grad_fn=)\n", + "tensor(0.1369, grad_fn=)\n", + "tensor(0.2340, grad_fn=)\n", + "tensor(0.1323, grad_fn=)\n", + "tensor(0.2564, grad_fn=)\n", + "tensor(0.2272, grad_fn=)\n", + "tensor(0.2023, grad_fn=)\n", + "tensor(0.1083, grad_fn=)\n", + "tensor(0.1650, grad_fn=)\n", + "tensor(0.2090, grad_fn=)\n", + "tensor(0.1944, grad_fn=)\n", + "tensor(0.1606, grad_fn=)\n", + "tensor(0.0920, grad_fn=)\n", + "tensor(0.0836, grad_fn=)\n", + "tensor(0.1067, grad_fn=)\n", + "tensor(0.1286, grad_fn=)\n", + "tensor(0.0456, grad_fn=)\n", + "tensor(0.1420, grad_fn=)\n", + "tensor(0.1227, grad_fn=)\n", + "tensor(0.1019, grad_fn=)\n", + "tensor(0.1073, grad_fn=)\n", + "tensor(0.0980, grad_fn=)\n", + "tensor(0.1237, grad_fn=)\n", + "tensor(0.1317, grad_fn=)\n", + "tensor(0.0849, grad_fn=)\n", + "tensor(0.0775, grad_fn=)\n", + "tensor(0.1032, grad_fn=)\n", + "tensor(0.0905, grad_fn=)\n", + "tensor(0.0317, grad_fn=)\n", + "tensor(0.1896, grad_fn=)\n", + "tensor(0.0413, grad_fn=)\n", + "tensor(0.0800, grad_fn=)\n", + "tensor(0.0905, grad_fn=)\n", + "tensor(0.1413, grad_fn=)\n", + "tensor(0.1100, grad_fn=)\n", + "tensor(0.1135, grad_fn=)\n", + "tensor(0.0593, grad_fn=)\n", + "tensor(0.0718, grad_fn=)\n", + "tensor(0.0811, grad_fn=)\n", + "tensor(0.0955, grad_fn=)\n", + "tensor(0.1069, grad_fn=)\n", + "tensor(0.0722, grad_fn=)\n", + "tensor(0.1101, grad_fn=)\n", + "tensor(0.1197, grad_fn=)\n", + "tensor(0.1142, grad_fn=)\n", + "tensor(0.0625, grad_fn=)\n", + "tensor(0.0421, grad_fn=)\n", + "[0.9892426913523349, 0.9987359198998748, 0.9692307692307692, 0.955640362225097, 0.987597977243995]\nacc:0.9800895439904143\nEpoch: 1/10\n----------\n", + "tensor(0.0700, grad_fn=)\n", + "tensor(0.0581, grad_fn=)\n", + "tensor(0.0917, grad_fn=)\n", + "tensor(0.0679, grad_fn=)\n", + "tensor(0.1409, grad_fn=)\n", + "tensor(0.0351, grad_fn=)\n" + ], + "output_type": "stream" + }, + { + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_validate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(m, train_loader, validate_loader, criterion, optimizer, epochs)\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Epoch: {}/{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"-\"\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_model_single_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 89\u001b[0m \u001b[0maccurency\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalidate_model_single_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"acc:{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maccurency\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_model_single_epoch\u001b[0;34m(m, data_loader, criterion, optimizer)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 30\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/ml-diplom/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \"\"\"\n\u001b[0;32m--> 118\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/ml-diplom/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 91\u001b[0m Variable._execution_engine.run_backward(\n\u001b[1;32m 92\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m allow_unreachable=True) # allow_unreachable flag\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ], + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error" + } + ], + "source": [ + "train_model(model, data_train, data_validate, crit, optim, 10)" + ], "metadata": { "collapsed": false, "pycharm": { - "name": "#%%\n" + "name": "#%%\n", + "is_executing": false } } } diff --git a/properties.py b/properties.py index a873273..341fc5c 100644 --- a/properties.py +++ b/properties.py @@ -17,7 +17,7 @@ image_size = 224 num_workers = 5 - +labels_number = len(classes) # 5 train_size = 200 test_size = 100 batch_size = 8 From f79e423b0e0fdb54b31d0e660778b43a07fc5330 Mon Sep 17 00:00:00 2001 From: glcanvas Date: Thu, 7 Nov 2019 23:36:19 +0300 Subject: [PATCH 05/10] isic fix f-measure classifier --- model_processing/train_model.py | 44 ++++ model_processing/validate_model.py | 72 ++++++ model_settings.py | 0 notebooks/classifier.ipynb | 214 ++++++++++++++++++ notebooks/init.ipynb | 339 ----------------------------- 5 files changed, 330 insertions(+), 339 deletions(-) create mode 100644 model_processing/train_model.py create mode 100644 model_processing/validate_model.py delete mode 100644 model_settings.py create mode 100644 notebooks/classifier.ipynb delete mode 100644 notebooks/init.ipynb diff --git a/model_processing/train_model.py b/model_processing/train_model.py new file mode 100644 index 0000000..50fa0db --- /dev/null +++ b/model_processing/train_model.py @@ -0,0 +1,44 @@ +from torchvision import models +import torch.utils.data.dataset +import torch +import torch.nn as nn +from torch.optim.optimizer import Optimizer +from model_processing.validate_model import ValidateModel + + +class TrainModel: + def __init__(self, validator: ValidateModel, device='cpu'): + self.device = device + self.validator = validator + + def train_model(self, m: models.AlexNet, train_loader: torch.utils.data.DataLoader, + validate_loader: torch.utils.data.DataLoader, + criterion: nn.CrossEntropyLoss, + optimizer: Optimizer, epochs: int) -> models.AlexNet: + + self.validator.flush() + + for epoch in range(epochs): + print("Epoch: {}/{}".format(epoch + 1, epochs)) + m = self._train_model_single_epoch(m, train_loader, criterion, optimizer) + acc = self.validator.validate_model_single_epoch(m, validate_loader) + print("acc: {}".format(acc)) + print("-" * 20) + + best_model = m.load_state_dict(self.validator.best_model_weights) + print("best acc: {}".format(self.validator.best_accuracy)) + return best_model + + def _train_model_single_epoch(self, m: models.AlexNet, data_loader: torch.utils.data.DataLoader, + criterion: nn.CrossEntropyLoss, optimizer: Optimizer) -> models.AlexNet: + m.train() + for inputs, labels in data_loader: + inputs = inputs.to(self.device) + labels = labels.type(torch.FloatTensor).to(self.device) + optimizer.zero_grad() + + model_result = m(inputs) + loss = criterion(model_result, labels) + loss.backward() + optimizer.step() + return m diff --git a/model_processing/validate_model.py b/model_processing/validate_model.py new file mode 100644 index 0000000..960f716 --- /dev/null +++ b/model_processing/validate_model.py @@ -0,0 +1,72 @@ +from torchvision import models +import torch.utils.data.dataset +import torch +import copy + + +class ValidateModel: + def __init__(self, labels_number: int, device='cpu'): + self.device = device + self.labels_number = labels_number + + self.iterate_accuracy = [] + self.best_accuracy = 0.0 + self.best_model_weights = None + + def validate_model_single_epoch(self, m: models.AlexNet, data_loader: torch.utils.data.DataLoader) -> float: + + with torch.set_grad_enabled(False): + confusion_matrix = torch \ + .tensor([[[0 for _ in range(2)] for _ in range(2)] for _ in range(self.labels_number)]) \ + .to(self.device) + + m.eval() + for inputs, labels in data_loader: + inputs = inputs.to(self.device) + labels = labels.to(self.device) + # trust result + for t in labels: + for idx, v in enumerate(t): + confusion_matrix[idx][v.item()][v.item()] += 1 + + model_result = m(inputs) + # all vectors size 5 + for i, t in enumerate(model_result): + # all classes + for j, v in enumerate(t): + dst_0 = abs(v.item()) + dst_1 = abs(1 - v.item()) + trust = labels[i][j].item() + model_answer = 1 if dst_1 < dst_0 else 0 + confusion_matrix[j][model_answer][trust] += 1 + + res = sum([self.__calculate_f_measure(i) for i in confusion_matrix]) / self.labels_number + if res > self.best_accuracy: + self.best_accuracy = res + self.best_model_weights = copy.deepcopy(m.state_dict()) + self.iterate_accuracy.append(res) + return res + + def flush(self): + self.iterate_accuracy = [] + self.best_accuracy = 0.0 + self.best_model_weights = None + + @staticmethod + def __calculate_f_measure(conf_matrix: torch.Tensor) -> float: + precision_w = 0 + recall_w = 0 + sum_all = conf_matrix.sum().item() + + for i, t in enumerate(conf_matrix): + c = 0 + p = 0 + for j, _ in enumerate(t): + c += conf_matrix[i][j].item() + p += conf_matrix[j][i].item() + if p == 0: + precision_w += 0 + else: + precision_w += ((conf_matrix[i][i].item() * c) / p) / sum_all + recall_w += conf_matrix[i][i].item() / sum_all + return (2 * (precision_w * recall_w)) / (precision_w + recall_w) diff --git a/model_settings.py b/model_settings.py deleted file mode 100644 index e69de29..0000000 diff --git a/notebooks/classifier.ipynb b/notebooks/classifier.ipynb new file mode 100644 index 0000000..d4ea504 --- /dev/null +++ b/notebooks/classifier.ipynb @@ -0,0 +1,214 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "outputs": [ + { + "name": "stdout", + "text": [ + "True\nGeForce GTX 1060 with Max-Q Design\nprepared data is exists\n" + ], + "output_type": "stream" + } + ], + "source": [ + "from torchvision import models\n", + "import torch.utils.data.dataset\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optimize\n", + "import image_loader as il\n", + "import properties as pr\n", + "from torch.multiprocessing import Pool\n", + "import os \n", + "\n", + "device = 'cuda' if torch.cuda.is_available() else \"cpu\"\n", + "\n", + "if device == 'cuda':\n", + " torch.multiprocessing.set_start_method('spawn', force=True)\n", + " torch.set_default_tensor_type('torch.cuda.FloatTensor')\n", + " \n", + "print(torch.cuda.is_available())\n", + "print( torch.cuda.get_device_name(0))\n", + "\n", + "if os.path.isdir(pr.tensor_train_path) and os.path.isdir(pr.tensor_validate_path):\n", + " print(\"prepared data is exists\")\n", + " train_tensor = il.load_tensors(pr.tensor_train_path)\n", + " validate_tensor = il.load_tensors(pr.tensor_validate_path)\n", + "else:\n", + " print(\"prepared data doesn't exists\")\n", + " r = il.map_inputs_labels(pr.data_inputs_path, pr.data_labels_path)\n", + " train, validate = il.split_set(r, pr.train_size, pr.test_size)\n", + " il.prepare_data(train, validate)\n", + " for i in train:\n", + " il.save_data(pr.tensor_train_path, i)\n", + " for j in validate:\n", + " il.save_data(pr.tensor_validate_path, j)\n", + " \n", + " train_tensor = il.load_tensors(pr.tensor_train_path)\n", + " validate_tensor = il.load_tensors(pr.tensor_validate_path)\n", + "\"\"\"\n", + "train_tensor = list(map(il.apply_resize, train_tensor))\n", + "validate_tensor = list(map(il.apply_resize, validate_tensor))\n", + "\n", + "for i in train_tensor:\n", + " il.save_data(pr.tensor_train_path, i)\n", + "for j in validate_tensor:\n", + " il.save_data(pr.tensor_validate_path, j)\n", + "\"\"\"\n", + "\n", + "train_tensor = list(map(il.remove_empty_mask, train_tensor))\n", + "validate_tensor = list(map(il.remove_empty_mask, validate_tensor))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": false + } + } + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "def get_model_parameters_to_update(m:models.AlexNet) -> list:\n", + " params = []\n", + " for name, param in m.named_parameters():\n", + " # maybe here better check require gradient back propagation\n", + " # and then update\n", + " # print(\"name={}\".format(name))\n", + " params.append(param)\n", + " return params\n", + "\n", + "def set_model_last_layer(m:models.AlexNet) -> models.AlexNet:\n", + " num_features = m.classifier[6].in_features\n", + " m.classifier[6] = nn.Linear(num_features, pr.labels_number) # here 2 for each label \n", + " return m" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": false + } + } + }, + { + "cell_type": "code", + "execution_count": 3, + "outputs": [], + "source": [ + "### EXECUTE ONLY ONCE!!!!!!!!!!!\n", + "model = models.alexnet(pretrained=True, progress=False)\n", + "model = set_model_last_layer(model)\n", + "\n", + "parameters = get_model_parameters_to_update(model)\n", + "data_train = torch.utils.data.DataLoader(il.CustomDataset(train_tensor), batch_size=4, shuffle=True, num_workers=4)\n", + "data_validate = torch.utils.data.DataLoader(il.CustomDataset(validate_tensor), batch_size=4, shuffle=True, num_workers=4)\n", + "crit = nn.SmoothL1Loss()\n", + "optim = optimize.SGD(parameters, lr=0.001, momentum=0.9)\n" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% \n", + "is_executing": false + } + } + }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [], + "source": [ + "from model_processing.train_model import TrainModel\n", + "\n", + "from model_processing.validate_model import ValidateModel" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": false + } + } + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [ + { + "name": "stdout", + "text": [ + "Epoch: 1/10\n", + "acc: 0.9158437994538648\n--------------------\nEpoch: 2/10\n", + "acc: 0.9045918345978704\n--------------------\nEpoch: 3/10\n", + "acc: 0.9134134785693929\n--------------------\nEpoch: 4/10\n", + "acc: 0.9201382334890909\n--------------------\nEpoch: 5/10\n", + "acc: 0.9065144023935922\n--------------------\nEpoch: 6/10\n", + "acc: 0.9097917530667047\n--------------------\nEpoch: 7/10\n", + "acc: 0.9147743398809975\n--------------------\nEpoch: 8/10\n", + "acc: 0.8996274051075769\n--------------------\nEpoch: 9/10\n", + "acc: 0.9127863482955301\n--------------------\nEpoch: 10/10\n", + "acc: 0.9159036728674325\n--------------------\nbest acc: {} 0.9201382334890909\n" + ], + "output_type": "stream" + }, + { + "data": { + "text/plain": "" + }, + "metadata": {}, + "output_type": "execute_result", + "execution_count": 5 + } + ], + "source": [ + "validate_model = ValidateModel(pr.labels_number, device)\n", + "train_model = TrainModel(validate_model, device)\n", + "\n", + "train_model.train_model(model, data_train,data_validate, crit, optim, 10)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": false + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + }, + "pycharm": { + "stem_cell": { + "cell_type": "raw", + "source": [], + "metadata": { + "collapsed": false + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/notebooks/init.ipynb b/notebooks/init.ipynb deleted file mode 100644 index 33884b2..0000000 --- a/notebooks/init.ipynb +++ /dev/null @@ -1,339 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "outputs": [ - { - "name": "stderr", - "text": [ - "/home/nikita/anaconda3/envs/ml-diplom/lib/python3.7/site-packages/torchvision/transforms/transforms.py:210: UserWarning: The use of the transforms.Scale transform is deprecated, please use transforms.Resize instead.\n warnings.warn(\"The use of the transforms.Scale transform is deprecated, \" +\n" - ], - "output_type": "stream" - }, - { - "name": "stdout", - "text": [ - "True\nGeForce GTX 1060 with Max-Q Design\n" - ], - "output_type": "stream" - } - ], - "source": [ - "from torchvision import models\n", - "import torch.utils.data.dataset\n", - "import torch\n", - "import torch.nn as nn\n", - "from torch.optim.optimizer import Optimizer\n", - "import torch.optim as optimize\n", - "import image_loader as il\n", - "import properties as pr\n", - "import copy\n", - "from torch.multiprocessing import Pool, Process, set_start_method\n", - "\n", - "#torch.multiprocessing.set_start_method('spawn', force=True)\n", - "device = 'cpu'\n", - "#torch.set_default_tensor_type('torch.cuda.FloatTensor')\n", - " \n", - "print(torch.cuda.is_available())\n", - "print( torch.cuda.get_device_name(0))\n", - "\"\"\"\n", - "r = il.map_inputs_labels(pr.data_inputs_path, pr.data_labels_path)\n", - "\n", - "train, validate = il.split_set(r, pr.train_size, pr.test_size)\n", - "\n", - "il.prepare_data(train, validate)\n", - "\n", - "for i in train:\n", - " il.save_data(pr.tensor_train_path, i)\n", - "for j in validate:\n", - " il.save_data(pr.tensor_validate_path, j)\n", - "\"\"\"\n", - "\n", - "train_tensor = il.load_tensors(pr.tensor_train_path)\n", - "validate_tensor = il.load_tensors(pr.tensor_validate_path)\n", - "\n", - "\"\"\"\n", - "train_tensor = list(map(il.apply_resize, train_tensor))\n", - "validate_tensor = list(map(il.apply_resize, validate_tensor))\n", - "\n", - "for i in train_tensor:\n", - " il.save_data(pr.tensor_train_path, i)\n", - "for j in validate_tensor:\n", - " il.save_data(pr.tensor_validate_path, j)\n", - "\"\"\"\n", - "\n", - "train_tensor = list(map(il.remove_empty_mask, train_tensor))\n", - "validate_tensor = list(map(il.remove_empty_mask, validate_tensor))" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } - }, - { - "cell_type": "code", - "execution_count": 8, - "outputs": [], - "source": [ - "def get_model_parameters_to_update(m:models.AlexNet) -> list:\n", - " params = []\n", - " for name, param in m.named_parameters():\n", - " # maybe here better check require gradient back propagation\n", - " # and then update\n", - " # print(\"name={}\".format(name))\n", - " params.append(param)\n", - " return params\n", - "\n", - "def set_model_last_layer(m:models.AlexNet) -> models.AlexNet:\n", - " num_features = m.classifier[6].in_features\n", - " m.classifier[6] = nn.Linear(num_features, pr.labels_number) # here 2 for each label \n", - " return m\n", - "\n", - "def train_model_single_epoch(m:models.AlexNet, data_loader:torch.utils.data.DataLoader, criterion:nn.CrossEntropyLoss, \n", - " optimizer:Optimizer) -> models.AlexNet:\n", - " m.train()\n", - " for inputs, labels in data_loader:\n", - " inputs = inputs.to(device)\n", - " labels = labels.type(torch.FloatTensor).to(device)\n", - " optimizer.zero_grad()\n", - " \n", - " model_result = m(inputs)\n", - " #print(model_result)\n", - " #print(\"-\" * 20)\n", - " #print(labels)\n", - " loss = criterion(model_result, labels)\n", - " print(loss)\n", - " loss.backward()\n", - " optimizer.step()\n", - " return m\n", - "\n", - "def validate_model_single_epoch(m:models.AlexNet, data_loader:torch.utils.data.DataLoader) -> float:\n", - " confusion_matrix = torch\\\n", - " .tensor([[[0 for _ in range(2)] for _ in range(2)] for _ in range(pr.labels_number)])\\\n", - " .to(device)\n", - " \n", - " m.eval()\n", - " for inputs, labels in data_loader:\n", - " inputs = inputs.to(device)\n", - " labels = labels.to(device)\n", - " # trust result\n", - " for t in labels:\n", - " for idx, v in enumerate(t):\n", - " confusion_matrix[idx][v.item()][v.item()] += 1\n", - "\n", - " model_result = m(inputs)\n", - " # all vectors size 5\n", - " for i, t in enumerate(model_result):\n", - " # all classes\n", - " for j, v in enumerate(t):\n", - " dst_0 = abs(v.item())\n", - " dst_1 = abs(1 - v.item())\n", - " trust = labels[i][j].item()\n", - " model_answer = 1 if dst_1 < dst_0 else 0 \n", - " confusion_matrix[j][model_answer][trust] += 1 \n", - " \n", - " print([calculate_f_measure(i) for i in confusion_matrix])\n", - " return sum([calculate_f_measure(i) for i in confusion_matrix]) / pr.labels_number\n", - "\n", - "def calculate_f_measure(conf_matrix:torch.Tensor) -> float:\n", - " precision_w = 0\n", - " recall_w = 0\n", - " sum_all = conf_matrix.sum().item()\n", - " \n", - " for i, t in enumerate(conf_matrix):\n", - " c = 0\n", - " p = 0\n", - " for j, _ in enumerate(t):\n", - " c += conf_matrix[i][j].item()\n", - " p += conf_matrix[j][i].item()\n", - " if p == 0:\n", - " precision_w += 0\n", - " else: \n", - " precision_w += ((conf_matrix[i][i].item() * c) / p) / sum_all\n", - " recall_w += conf_matrix[i][i].item() / sum_all\n", - " return (2 * (precision_w * recall_w)) / (precision_w + recall_w)\n", - " \n", - "\n", - "def train_model(m:models.AlexNet, train_loader:torch.utils.data.DataLoader, validate_loader:torch.utils.data.DataLoader,\n", - " criterion:nn.CrossEntropyLoss, \n", - " optimizer:Optimizer, epochs:int) -> models.AlexNet:\n", - " best_model_state = copy.deepcopy(model.state_dict())\n", - " best_acc = 0.0\n", - " for epoch in range(epochs):\n", - " print(\"Epoch: {}/{}\".format(epoch, epochs))\n", - " print(\"-\" * 10)\n", - " m = train_model_single_epoch(m, train_loader, criterion, optimizer)\n", - " accurency = validate_model_single_epoch(m, validate_loader)\n", - " print(\"acc:{}\".format(accurency))\n", - " if accurency > best_acc:\n", - " best_acc = accurency\n", - " best_model_state = copy.deepcopy(m.state_dict())\n", - " \n", - " best_model = m.load_state_dict(best_model_state)\n", - " print(best_acc)\n", - " return best_model\n", - " \n", - " " - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } - }, - { - "cell_type": "code", - "execution_count": 9, - "outputs": [], - "source": [ - "### EXECUTE ONLY ONCE!!!!!!!!!!!\n", - "model = models.alexnet(pretrained=True, progress=False)\n", - "model = set_model_last_layer(model)\n", - "\n", - "parameters = get_model_parameters_to_update(model)\n", - "data_train = torch.utils.data.DataLoader(il.CustomDataset(train_tensor), batch_size=4, shuffle=True, num_workers=4)\n", - "data_validate = torch.utils.data.DataLoader(il.CustomDataset(validate_tensor), batch_size=4, shuffle=True, num_workers=4)\n", - "crit = nn.SmoothL1Loss()\n", - "optim = optimize.SGD(parameters, lr=0.001, momentum=0.9)\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% \n", - "is_executing": false - } - } - }, - { - "cell_type": "code", - "execution_count": 10, - "outputs": [ - { - "name": "stdout", - "text": [ - "Epoch: 0/10\n----------\n", - "tensor(0.6407, grad_fn=)\n", - "tensor(0.3992, grad_fn=)\n", - "tensor(0.1331, grad_fn=)\n", - "tensor(0.1047, grad_fn=)\n", - "tensor(0.1369, grad_fn=)\n", - "tensor(0.2340, grad_fn=)\n", - "tensor(0.1323, grad_fn=)\n", - "tensor(0.2564, grad_fn=)\n", - "tensor(0.2272, grad_fn=)\n", - "tensor(0.2023, grad_fn=)\n", - "tensor(0.1083, grad_fn=)\n", - "tensor(0.1650, grad_fn=)\n", - "tensor(0.2090, grad_fn=)\n", - "tensor(0.1944, grad_fn=)\n", - "tensor(0.1606, grad_fn=)\n", - "tensor(0.0920, grad_fn=)\n", - "tensor(0.0836, grad_fn=)\n", - "tensor(0.1067, grad_fn=)\n", - "tensor(0.1286, grad_fn=)\n", - "tensor(0.0456, grad_fn=)\n", - "tensor(0.1420, grad_fn=)\n", - "tensor(0.1227, grad_fn=)\n", - "tensor(0.1019, grad_fn=)\n", - "tensor(0.1073, grad_fn=)\n", - "tensor(0.0980, grad_fn=)\n", - "tensor(0.1237, grad_fn=)\n", - "tensor(0.1317, grad_fn=)\n", - "tensor(0.0849, grad_fn=)\n", - "tensor(0.0775, grad_fn=)\n", - "tensor(0.1032, grad_fn=)\n", - "tensor(0.0905, grad_fn=)\n", - "tensor(0.0317, grad_fn=)\n", - "tensor(0.1896, grad_fn=)\n", - "tensor(0.0413, grad_fn=)\n", - "tensor(0.0800, grad_fn=)\n", - "tensor(0.0905, grad_fn=)\n", - "tensor(0.1413, grad_fn=)\n", - "tensor(0.1100, grad_fn=)\n", - "tensor(0.1135, grad_fn=)\n", - "tensor(0.0593, grad_fn=)\n", - "tensor(0.0718, grad_fn=)\n", - "tensor(0.0811, grad_fn=)\n", - "tensor(0.0955, grad_fn=)\n", - "tensor(0.1069, grad_fn=)\n", - "tensor(0.0722, grad_fn=)\n", - "tensor(0.1101, grad_fn=)\n", - "tensor(0.1197, grad_fn=)\n", - "tensor(0.1142, grad_fn=)\n", - "tensor(0.0625, grad_fn=)\n", - "tensor(0.0421, grad_fn=)\n", - "[0.9892426913523349, 0.9987359198998748, 0.9692307692307692, 0.955640362225097, 0.987597977243995]\nacc:0.9800895439904143\nEpoch: 1/10\n----------\n", - "tensor(0.0700, grad_fn=)\n", - "tensor(0.0581, grad_fn=)\n", - "tensor(0.0917, grad_fn=)\n", - "tensor(0.0679, grad_fn=)\n", - "tensor(0.1409, grad_fn=)\n", - "tensor(0.0351, grad_fn=)\n" - ], - "output_type": "stream" - }, - { - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_validate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(m, train_loader, validate_loader, criterion, optimizer, epochs)\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Epoch: {}/{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"-\"\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_model_single_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 89\u001b[0m \u001b[0maccurency\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalidate_model_single_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"acc:{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maccurency\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_model_single_epoch\u001b[0;34m(m, data_loader, criterion, optimizer)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 30\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/ml-diplom/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \"\"\"\n\u001b[0;32m--> 118\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/ml-diplom/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 91\u001b[0m Variable._execution_engine.run_backward(\n\u001b[1;32m 92\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m allow_unreachable=True) # allow_unreachable flag\n\u001b[0m\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ], - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error" - } - ], - "source": [ - "train_model(model, data_train, data_validate, crit, optim, 10)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "source": [], - "metadata": { - "collapsed": false - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} \ No newline at end of file From eb36c9515af0e279d500aa8040bbe46d941d89ba Mon Sep 17 00:00:00 2001 From: glcanvas Date: Fri, 8 Nov 2019 00:23:32 +0300 Subject: [PATCH 06/10] merge --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index e9f6e55..2579cf9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,3 @@ такс, тут выяснилось (хорошо, что заранее) что у одной картинки может быть несколько ответов поэтому давайте для каждой тренировчной картинки сделаем набор из лэйблов -затем оставим только те лэйблы у которых лэйбл-карта не 0 - -но как тогда говорить верный ли ответ ? -2^5 вариантов выхода ? -- sounds good \ No newline at end of file +затем оставим только те лэйблы у которых лэйбл-карта не 0 \ No newline at end of file From 8721abf7cd33e9de129563bb3f8077df63e01783 Mon Sep 17 00:00:00 2001 From: glcanvas Date: Sun, 10 Nov 2019 21:28:40 +0300 Subject: [PATCH 07/10] fix `to` in cell --- image_loader.py | 1 - main.py | 1 - notebooks/classifier.ipynb | 45 ++++++++++++++++++++++++-------------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/image_loader.py b/image_loader.py index 608c20a..dfd0c53 100644 --- a/image_loader.py +++ b/image_loader.py @@ -68,7 +68,6 @@ def split_set(values: list, test_size: int, validate_size: int): def map_cell(cell: dict): for i in classes: cell[i] = composite(Image.open(cell[i])) - cell[i] = cell[i].to cell[field_train] = composite(Image.open(cell[field_train])) return cell diff --git a/main.py b/main.py index 3b1ab89..e69de29 100644 --- a/main.py +++ b/main.py @@ -1 +0,0 @@ -print("sss") \ No newline at end of file diff --git a/notebooks/classifier.ipynb b/notebooks/classifier.ipynb index d4ea504..3a60e48 100644 --- a/notebooks/classifier.ipynb +++ b/notebooks/classifier.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "outputs": [ { "name": "stdout", @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "outputs": [], "source": [ "def get_model_parameters_to_update(m:models.AlexNet) -> list:\n", @@ -98,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "outputs": [], "source": [ "### EXECUTE ONLY ONCE!!!!!!!!!!!\n", @@ -121,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "outputs": [], "source": [ "from model_processing.train_model import TrainModel\n", @@ -138,22 +138,22 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "outputs": [ { "name": "stdout", "text": [ "Epoch: 1/10\n", - "acc: 0.9158437994538648\n--------------------\nEpoch: 2/10\n", - "acc: 0.9045918345978704\n--------------------\nEpoch: 3/10\n", - "acc: 0.9134134785693929\n--------------------\nEpoch: 4/10\n", - "acc: 0.9201382334890909\n--------------------\nEpoch: 5/10\n", - "acc: 0.9065144023935922\n--------------------\nEpoch: 6/10\n", - "acc: 0.9097917530667047\n--------------------\nEpoch: 7/10\n", - "acc: 0.9147743398809975\n--------------------\nEpoch: 8/10\n", - "acc: 0.8996274051075769\n--------------------\nEpoch: 9/10\n", - "acc: 0.9127863482955301\n--------------------\nEpoch: 10/10\n", - "acc: 0.9159036728674325\n--------------------\nbest acc: {} 0.9201382334890909\n" + "acc: 0.9146376663058691\n--------------------\nEpoch: 2/10\n", + "acc: 0.9062037831070547\n--------------------\nEpoch: 3/10\n", + "acc: 0.9008440415733728\n--------------------\nEpoch: 4/10\n", + "acc: 0.904348759077464\n--------------------\nEpoch: 5/10\n", + "acc: 0.9190306151883995\n--------------------\nEpoch: 6/10\n", + "acc: 0.9155057995175717\n--------------------\nEpoch: 7/10\n", + "acc: 0.913126154190364\n--------------------\nEpoch: 8/10\n", + "acc: 0.9064585115767398\n--------------------\nEpoch: 9/10\n", + "acc: 0.9040153158076828\n--------------------\nEpoch: 10/10\n", + "acc: 0.9081597647843322\n--------------------\nbest acc: 0.9190306151883995\n" ], "output_type": "stream" }, @@ -163,7 +163,7 @@ }, "metadata": {}, "output_type": "execute_result", - "execution_count": 5 + "execution_count": 6 } ], "source": [ @@ -179,6 +179,19 @@ "is_executing": false } } + }, + { + "cell_type": "code", + "execution_count": 6, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n", + "is_executing": false + } + } } ], "metadata": { From 40923ef90ea55e8bdb481abf4eae183c75374c96 Mon Sep 17 00:00:00 2001 From: glcanvas Date: Fri, 22 Nov 2019 13:20:16 +0300 Subject: [PATCH 08/10] remove idea files --- .gitignore | 3 +++ .idea/.gitignore | 2 -- .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 7 ------- .idea/ml-diplom.iml | 16 ---------------- .idea/modules.xml | 8 -------- .idea/other.xml | 6 ------ .idea/vcs.xml | 6 ------ heh2 | 0 requirements.txt | 4 ---- 10 files changed, 3 insertions(+), 55 deletions(-) create mode 100644 .gitignore delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/ml-diplom.iml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/other.xml delete mode 100644 .idea/vcs.xml delete mode 100644 heh2 delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a8dfff --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# file: ~/.gitignore_global +.DS_Store +.idea diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 5c98b42..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Default ignored files -/workspace.xml \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index a184fa6..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/ml-diplom.iml b/.idea/ml-diplom.iml deleted file mode 100644 index e1c70e3..0000000 --- a/.idea/ml-diplom.iml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index cc4ee99..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/other.xml b/.idea/other.xml deleted file mode 100644 index 68993fb..0000000 --- a/.idea/other.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/heh2 b/heh2 deleted file mode 100644 index e69de29..0000000 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a035fdc..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -scikit-learn -PIL -torchvision -torch From 6cbf77564ca4e5af9b9bebc8b9d6c58e6660745a Mon Sep 17 00:00:00 2001 From: glcanvas Date: Fri, 22 Nov 2019 13:43:47 +0300 Subject: [PATCH 09/10] add f1_score --- model_processing/validate_model.py | 37 ++++--------------- notebooks/classifier.ipynb | 57 +++++++++++++----------------- 2 files changed, 31 insertions(+), 63 deletions(-) diff --git a/model_processing/validate_model.py b/model_processing/validate_model.py index 960f716..ac00f43 100644 --- a/model_processing/validate_model.py +++ b/model_processing/validate_model.py @@ -2,6 +2,7 @@ import torch.utils.data.dataset import torch import copy +from sklearn.metrics import f1_score class ValidateModel: @@ -16,18 +17,12 @@ def __init__(self, labels_number: int, device='cpu'): def validate_model_single_epoch(self, m: models.AlexNet, data_loader: torch.utils.data.DataLoader) -> float: with torch.set_grad_enabled(False): - confusion_matrix = torch \ - .tensor([[[0 for _ in range(2)] for _ in range(2)] for _ in range(self.labels_number)]) \ - .to(self.device) - + trust_answer = [[] for _ in range(self.labels_number)] + model_answer = [[] for _ in range(self.labels_number)] m.eval() for inputs, labels in data_loader: inputs = inputs.to(self.device) labels = labels.to(self.device) - # trust result - for t in labels: - for idx, v in enumerate(t): - confusion_matrix[idx][v.item()][v.item()] += 1 model_result = m(inputs) # all vectors size 5 @@ -37,10 +32,11 @@ def validate_model_single_epoch(self, m: models.AlexNet, data_loader: torch.util dst_0 = abs(v.item()) dst_1 = abs(1 - v.item()) trust = labels[i][j].item() - model_answer = 1 if dst_1 < dst_0 else 0 - confusion_matrix[j][model_answer][trust] += 1 + ma = 1 if dst_1 < dst_0 else 0 + trust_answer[j].append(trust) + model_answer[j].append(ma) - res = sum([self.__calculate_f_measure(i) for i in confusion_matrix]) / self.labels_number + res = sum([f1_score(t, m) for t, m in zip(trust_answer, model_answer)]) / self.labels_number if res > self.best_accuracy: self.best_accuracy = res self.best_model_weights = copy.deepcopy(m.state_dict()) @@ -51,22 +47,3 @@ def flush(self): self.iterate_accuracy = [] self.best_accuracy = 0.0 self.best_model_weights = None - - @staticmethod - def __calculate_f_measure(conf_matrix: torch.Tensor) -> float: - precision_w = 0 - recall_w = 0 - sum_all = conf_matrix.sum().item() - - for i, t in enumerate(conf_matrix): - c = 0 - p = 0 - for j, _ in enumerate(t): - c += conf_matrix[i][j].item() - p += conf_matrix[j][i].item() - if p == 0: - precision_w += 0 - else: - precision_w += ((conf_matrix[i][i].item() * c) / p) / sum_all - recall_w += conf_matrix[i][i].item() / sum_all - return (2 * (precision_w * recall_w)) / (precision_w + recall_w) diff --git a/notebooks/classifier.ipynb b/notebooks/classifier.ipynb index 3a60e48..c4369b9 100644 --- a/notebooks/classifier.ipynb +++ b/notebooks/classifier.ipynb @@ -2,12 +2,19 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "outputs": [ + { + "name": "stderr", + "text": [ + "/home/nikita/anaconda3/envs/ml-diplom/lib/python3.7/site-packages/torchvision/transforms/transforms.py:210: UserWarning: The use of the transforms.Scale transform is deprecated, please use transforms.Resize instead.\n warnings.warn(\"The use of the transforms.Scale transform is deprecated, \" +\n" + ], + "output_type": "stream" + }, { "name": "stdout", "text": [ - "True\nGeForce GTX 1060 with Max-Q Design\nprepared data is exists\n" + "prepared data is exists\n" ], "output_type": "stream" } @@ -28,9 +35,6 @@ "if device == 'cuda':\n", " torch.multiprocessing.set_start_method('spawn', force=True)\n", " torch.set_default_tensor_type('torch.cuda.FloatTensor')\n", - " \n", - "print(torch.cuda.is_available())\n", - "print( torch.cuda.get_device_name(0))\n", "\n", "if os.path.isdir(pr.tensor_train_path) and os.path.isdir(pr.tensor_validate_path):\n", " print(\"prepared data is exists\")\n", @@ -71,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "outputs": [], "source": [ "def get_model_parameters_to_update(m:models.AlexNet) -> list:\n", @@ -98,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "outputs": [], "source": [ "### EXECUTE ONLY ONCE!!!!!!!!!!!\n", @@ -121,7 +125,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "outputs": [], "source": [ "from model_processing.train_model import TrainModel\n", @@ -138,22 +142,22 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "outputs": [ { "name": "stdout", "text": [ "Epoch: 1/10\n", - "acc: 0.9146376663058691\n--------------------\nEpoch: 2/10\n", - "acc: 0.9062037831070547\n--------------------\nEpoch: 3/10\n", - "acc: 0.9008440415733728\n--------------------\nEpoch: 4/10\n", - "acc: 0.904348759077464\n--------------------\nEpoch: 5/10\n", - "acc: 0.9190306151883995\n--------------------\nEpoch: 6/10\n", - "acc: 0.9155057995175717\n--------------------\nEpoch: 7/10\n", - "acc: 0.913126154190364\n--------------------\nEpoch: 8/10\n", - "acc: 0.9064585115767398\n--------------------\nEpoch: 9/10\n", - "acc: 0.9040153158076828\n--------------------\nEpoch: 10/10\n", - "acc: 0.9081597647843322\n--------------------\nbest acc: 0.9190306151883995\n" + "acc: 0.851646648908066\n--------------------\nEpoch: 2/10\n", + "acc: 0.8464873758965359\n--------------------\nEpoch: 3/10\n", + "acc: 0.8395762580569832\n--------------------\nEpoch: 4/10\n", + "acc: 0.7985506170313421\n--------------------\nEpoch: 5/10\n", + "acc: 0.7958712400146775\n--------------------\nEpoch: 6/10\n", + "acc: 0.8139802083403304\n--------------------\nEpoch: 7/10\n", + "acc: 0.8478839503646756\n--------------------\nEpoch: 8/10\n", + "acc: 0.7804984863489599\n--------------------\nEpoch: 9/10\n", + "acc: 0.808924053468127\n--------------------\nEpoch: 10/10\n", + "acc: 0.7809302213536918\n--------------------\nbest acc: 0.851646648908066\n" ], "output_type": "stream" }, @@ -163,7 +167,7 @@ }, "metadata": {}, "output_type": "execute_result", - "execution_count": 6 + "execution_count": 5 } ], "source": [ @@ -179,19 +183,6 @@ "is_executing": false } } - }, - { - "cell_type": "code", - "execution_count": 6, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } } ], "metadata": { From 2e3e9ec4d644e880884c57ed82ac34bcfa4c1770 Mon Sep 17 00:00:00 2001 From: glcanvas Date: Fri, 22 Nov 2019 13:46:15 +0300 Subject: [PATCH 10/10] update .gitigonre --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 0a8dfff..3592172 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ # file: ~/.gitignore_global .DS_Store .idea +__pycache__ +model_processing/__pycache__ \ No newline at end of file