From 7a8085463afbdb53673152fce6bf361d5987dfbb Mon Sep 17 00:00:00 2001 From: "Paolo \"Nusco\" Perrotta" Date: Mon, 7 Jan 2019 15:13:04 +0100 Subject: [PATCH] Add English translation of MNIST notebook --- notebooks/mlp_en.ipynb | 418 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 418 insertions(+) create mode 100644 notebooks/mlp_en.ipynb diff --git a/notebooks/mlp_en.ipynb b/notebooks/mlp_en.ipynb new file mode 100644 index 0000000..4384fb3 --- /dev/null +++ b/notebooks/mlp_en.ipynb @@ -0,0 +1,418 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Multilayer Perceptron" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Based on:\n", + "\n", + "- http://gluon.mxnet.io/chapter03_deep-neural-networks/mlp-scratch.html" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "require 'mxnet'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@data_ctx = MXNet.cpu\n", + "@model_ctx = MXNet.cpu\n", + "#@model_ctx = MXNet.gpu" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Download the data (unless it's already there):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "unless File.exist?('train-images-idx3-ubyte') &&\n", + " File.exist?('train-labels-idx1-ubyte')\n", + " system(\"wget http://data.mxnet.io/mxnet/data/mnist.zip\")\n", + " system(\"unzip -x mnist.zip\")\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "num_inputs = 784\n", + "num_outputs = 10\n", + "batch_size = 64\n", + "num_examples = 60000\n", + "\n", + "train_iter = MXNet::IO::MNISTIter.new(\n", + " batch_size: batch_size,\n", + " shuffle: true)\n", + "test_iter = MXNet::IO::MNISTIter.new(\n", + " image: 't10k-images-idx3-ubyte',\n", + " label: 't10k-labels-idx1-ubyte',\n", + " batch_size: batch_size,\n", + " shuffle: false)\n", + "nil" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set Parameters" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the neural network's parameters:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#######################\n", + "# Set some constants so it's easy to modify the network later\n", + "#######################\n", + "num_hidden = 256\n", + "weight_scale = 0.01\n", + "\n", + "#######################\n", + "# Allocate parameters for the first hidden layer\n", + "#######################\n", + "@w1 = MXNet::NDArray.random_normal(shape: [num_inputs, num_hidden], scale: weight_scale, ctx: @model_ctx)\n", + "@b1 = MXNet::NDArray.random_normal(shape: [num_hidden], scale: weight_scale, ctx: @model_ctx)\n", + "\n", + "#######################\n", + "# Allocate parameters for the second hidden layer\n", + "#######################\n", + "@w2 = MXNet::NDArray.random_normal(shape: [num_hidden, num_hidden], scale: weight_scale, ctx: @model_ctx)\n", + "@b2 = MXNet::NDArray.random_normal(shape: [num_hidden], scale: weight_scale, ctx: @model_ctx)\n", + "\n", + "#######################\n", + "# Allocate parameters for the output layer\n", + "#######################\n", + "@w3 = MXNet::NDArray.random_normal(shape: [num_hidden, num_outputs], scale: weight_scale, ctx: @model_ctx)\n", + "@b3 = MXNet::NDArray.random_normal(shape: [num_outputs], scale: weight_scale, ctx: @model_ctx)\n", + "\n", + "@params = [@w1, @b1, @w2, @b2, @w3, @b3]\n", + "nil" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@params.each do |param|\n", + " param.attach_grad\n", + "end\n", + "nil" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Activation Function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def relu(x)\n", + " MXNet::NDArray.maximum(x, MXNet::NDArray.zeros_like(x))\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Softmax" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def softmax(y_linear)\n", + " exp = MXNet::NDArray.exp(y_linear - MXNet::NDArray.max(y_linear))\n", + " partition = MXNet::NDArray.nansum(exp, axis: 0, exclude: true).reshape([-1, 1])\n", + " return exp / partition\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cross Entropy Loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def cross_entropy(y_hat, y)\n", + " return -MXNet::NDArray.nansum(y * MXNet::NDArray.log(y_hat), axis: 0, exclude: true)\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def softmax_cross_entropy(y_hat_linear, y)\n", + " return -MXNet::NDArray.nansum(y * MXNet::NDArray.log_softmax(y_hat_linear), axis: 0, exclude: true)\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model Definition" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def net(x)\n", + " # first hidden layer\n", + " h1_linear = MXNet::NDArray.dot(x, @w1) + @b1\n", + " h1 = relu(h1_linear)\n", + "\n", + " # second hidden layer\n", + " h2_linear = MXNet::NDArray.dot(h1, @w2) + @b2\n", + " h2 = relu(h2_linear)\n", + "\n", + " # output layer\n", + " y_hat_linear = MXNet::NDArray.dot(h2, @w3) + @b3\n", + " return y_hat_linear\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optimizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def sgd(params, lr)\n", + " params.each do |param|\n", + " param[0..-1] = param - lr * param.grad\n", + " end\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_accuracy(data_iter)\n", + " numerator = 0.0\n", + " denominator = 0.0\n", + " data_iter.each_with_index do |batch, i|\n", + " data = batch.data[0].as_in_context(@model_ctx).reshape([-1, 784])\n", + " label = batch.label[0].as_in_context(@model_ctx)\n", + " output = net(data)\n", + " predictions = MXNet::NDArray.argmax(output, axis: 1)\n", + " numerator += MXNet::NDArray.sum(predictions == label)\n", + " denominator += data.shape[0]\n", + " end\n", + " return (numerator / denominator).as_scalar\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "epochs = 10\n", + "learning_rate = 0.001\n", + "smoothing_constant = 0.01\n", + "\n", + "epochs.times do |e|\n", + " start = Time.now\n", + " cumulative_loss = 0.0\n", + " train_iter.each_with_index do |batch, i|\n", + " data = batch.data[0].as_in_context(@model_ctx).reshape([-1, 784])\n", + " label = batch.label[0].as_in_context(@model_ctx)\n", + " label_one_hot = MXNet::NDArray.one_hot(label, depth: 10)\n", + " loss = MXNet::Autograd.record do\n", + " output = net(data)\n", + " softmax_cross_entropy(output, label_one_hot)\n", + " end\n", + " loss.backward()\n", + " sgd(@params, learning_rate)\n", + " cumulative_loss += MXNet::NDArray.sum(loss).as_scalar\n", + " end\n", + " \n", + " test_accuracy = evaluate_accuracy(test_iter)\n", + " train_accuracy = evaluate_accuracy(train_iter)\n", + " duration = Time.now - start\n", + " puts \"Epoch #{e}. Loss: #{cumulative_loss/num_examples}, Train_acc #{train_accuracy}, Test_acc #{test_accuracy} (#{duration} sec)\"\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Classification" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "require 'chunky_png'\n", + "require 'base64'\n", + "\n", + "def imshow(ary)\n", + " height, width = ary.shape\n", + " fig = ChunkyPNG::Image.new(width, height, ChunkyPNG::Color::TRANSPARENT)\n", + " ary = ((ary - ary.min) / ary.max) * 255\n", + " 0.upto(height - 1) do |i|\n", + " 0.upto(width - 1) do |j|\n", + " v = ary[i, j].round\n", + " fig[j, i] = ChunkyPNG::Color.rgba(v, v, v, 255)\n", + " end\n", + " end\n", + "\n", + " src = 'data:image/png;base64,' + Base64.strict_encode64(fig.to_blob)\n", + " IRuby.display \"\", mime: 'text/html'\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define the funtion to do prediction\n", + "def model_predict(data)\n", + " output = net(data)\n", + " MXNet::NDArray.argmax(output, axis: 1)\n", + "end\n", + "\n", + "samples = 10\n", + "sample_iter = test_iter = MXNet::IO::MNISTIter.new(\n", + " image: 't10k-images-idx3-ubyte',\n", + " label: 't10k-labels-idx1-ubyte',\n", + " batch_size: samples,\n", + " shuffle: true)\n", + "sample_iter.each do |batch|\n", + " batch = sample_iter.next_batch\n", + " data = batch.data[0].as_in_context(@model_ctx)\n", + " label = batch.label[0]\n", + "\n", + " im = data.transpose(axes: [1, 0, 2, 3]).reshape([10*28, 28, 1])\n", + " imshow(im[0..-1, 0..-1, 0].to_narray)\n", + "\n", + " pred = model_predict(data.reshape([-1, 784]))\n", + " puts \"model predictions are: #{pred.inspect}\"\n", + " puts\n", + " puts \"true labels: #{label.inspect}\"\n", + " break\n", + "end" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Ruby 2.6.0", + "language": "ruby", + "name": "ruby" + }, + "language_info": { + "file_extension": ".rb", + "mimetype": "application/x-ruby", + "name": "ruby", + "version": "2.6.0" + }, + "toc": { + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "toc_cell": false, + "toc_position": {}, + "toc_section_display": "block", + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}