diff --git a/Run_train_tasks_in_Colab_Pro.ipynb b/Run_train_tasks_in_Colab_Pro.ipynb new file mode 100644 index 0000000..84ada36 --- /dev/null +++ b/Run_train_tasks_in_Colab_Pro.ipynb @@ -0,0 +1,5283 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "ViLBERT.ipynb", + "provenance": [], + "collapsed_sections": [], + "machine_shape": "hm" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "0rLYLqyWM8hI" + }, + "source": [ + "# ViLBERT fine-tuning / evaluation in Colab\n", + "- Operation checked at: Colab PRO GPU mode with large storage (>=approx 100GB)\n", + " - Similar environment with Linux and large storage should be fine, but not tested" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4fgEiawZNF7I" + }, + "source": [ + "## Module Setup\n", + "- https://github.com/facebookresearch/vilbert-multi-task" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZBTIBgNvQ2Si" + }, + "source": [ + "### requirements.txt" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "OAnQkpGoNBzL", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "89f5c38d-53da-474d-b1b1-c7198722fe5f" + }, + "source": [ + "# !conda create -n vilbert-mt python=3.6\n", + "# !conda activate vilbert-mt\n", + "!git clone --recursive https://github.com/facebookresearch/vilbert-multi-task.git\n", + "%cd vilbert-multi-task\n", + "# https://stackoverflow.com/questions/22250483/stop-pip-from-failing-on-single-package-when-installing-with-requirements-txt\n", + "!cat requirements.txt | sed -e '/^\\s*#.*$/d' -e '/^\\s*$/d' | xargs -n 1 pip install" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Cloning into 'vilbert-multi-task'...\n", + "remote: Enumerating objects: 2005, done.\u001b[K\n", + "remote: Counting objects: 100% (3/3), done.\u001b[K\n", + "remote: Compressing objects: 100% (3/3), done.\u001b[K\n", + "remote: Total 2005 (delta 0), reused 0 (delta 0), pack-reused 2002\u001b[K\n", + "Receiving objects: 100% (2005/2005), 117.16 MiB | 37.85 MiB/s, done.\n", + "Resolving deltas: 100% (1436/1436), done.\n", + "Submodule 'tools/refer' (https://github.com/lichengunc/refer.git) registered for path 'tools/refer'\n", + "Cloning into '/content/vilbert-multi-task/tools/refer'...\n", + "remote: Enumerating objects: 185, done. \n", + "remote: Counting objects: 100% (3/3), done. \n", + "remote: Compressing objects: 100% (3/3), done. \n", + "remote: Total 185 (delta 0), reused 0 (delta 0), pack-reused 182 \n", + "Receiving objects: 100% (185/185), 71.66 MiB | 58.89 MiB/s, done.\n", + "Resolving deltas: 100% (72/72), done.\n", + "Submodule path 'tools/refer': checked out 'd23ec5d8e84baf858c58af119799e26846aa5261'\n", + "/content/vilbert-multi-task\n", + "Collecting pytorch-transformers==1.0.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/40/b5/2d78e74001af0152ee61d5ad4e290aec9a1e43925b21df2dc74ec100f1ab/pytorch_transformers-1.0.0-py3-none-any.whl (137kB)\n", + "\u001b[K |████████████████████████████████| 143kB 5.3MB/s \n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from pytorch-transformers==1.0.0) (1.19.5)\n", + "Requirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from pytorch-transformers==1.0.0) (1.8.1+cu101)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from pytorch-transformers==1.0.0) (2019.12.20)\n", + "Collecting boto3\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/4b/a3/388f78a63b0545f30daf13e6542b0d850f621ff51ef721cf431eef609c87/boto3-1.17.59-py2.py3-none-any.whl (131kB)\n", + "\u001b[K |████████████████████████████████| 133kB 8.3MB/s \n", + "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from pytorch-transformers==1.0.0) (2.23.0)\n", + "Collecting sentencepiece\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n", + "\u001b[K |████████████████████████████████| 1.2MB 8.4MB/s \n", + "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from pytorch-transformers==1.0.0) (4.41.1)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=0.4.1->pytorch-transformers==1.0.0) (3.7.4.3)\n", + "Collecting s3transfer<0.5.0,>=0.4.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/63/d0/693477c688348654ddc21dcdce0817653a294aa43f41771084c25e7ff9c7/s3transfer-0.4.2-py2.py3-none-any.whl (79kB)\n", + "\u001b[K |████████████████████████████████| 81kB 8.0MB/s \n", + "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n", + " Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n", + "Collecting botocore<1.21.0,>=1.20.59\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/25/68/e91883d8f68183216fac7cfa855cd0d1b3336c3d4b82cd4b99591118bcc7/botocore-1.20.59-py2.py3-none-any.whl (7.4MB)\n", + "\u001b[K |████████████████████████████████| 7.4MB 12.0MB/s \n", + "\u001b[?25hRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->pytorch-transformers==1.0.0) (1.24.3)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->pytorch-transformers==1.0.0) (2.10)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->pytorch-transformers==1.0.0) (3.0.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->pytorch-transformers==1.0.0) (2020.12.5)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.7/dist-packages (from botocore<1.21.0,>=1.20.59->boto3->pytorch-transformers==1.0.0) (2.8.1)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.21.0,>=1.20.59->boto3->pytorch-transformers==1.0.0) (1.15.0)\n", + "\u001b[31mERROR: botocore 1.20.59 has requirement urllib3<1.27,>=1.25.4, but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n", + "Installing collected packages: jmespath, botocore, s3transfer, boto3, sentencepiece, pytorch-transformers\n", + "Successfully installed boto3-1.17.59 botocore-1.20.59 jmespath-0.10.0 pytorch-transformers-1.0.0 s3transfer-0.4.2 sentencepiece-0.1.95\n", + "Collecting numpy==1.16.4\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/fc/d1/45be1144b03b6b1e24f9a924f23f66b4ad030d834ad31fb9e5581bd328af/numpy-1.16.4-cp37-cp37m-manylinux1_x86_64.whl (17.3MB)\n", + "\u001b[K |████████████████████████████████| 17.3MB 132kB/s \n", + "\u001b[31mERROR: tensorflow 2.4.1 has requirement numpy~=1.19.2, but you'll have numpy 1.16.4 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: pyarrow 3.0.0 has requirement numpy>=1.16.6, but you'll have numpy 1.16.4 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: astropy 4.2.1 has requirement numpy>=1.17, but you'll have numpy 1.16.4 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n", + "\u001b[?25hInstalling collected packages: numpy\n", + " Found existing installation: numpy 1.19.5\n", + " Uninstalling numpy-1.19.5:\n", + " Successfully uninstalled numpy-1.19.5\n", + "Successfully installed numpy-1.16.4\n", + "Collecting lmdb==0.94\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/cb/31/5be8f436b56733d9e69c721c358502f4d77b627489a459978686be7db65f/lmdb-0.94.tar.gz (4.0MB)\n", + "\u001b[K |████████████████████████████████| 4.0MB 6.6MB/s \n", + "\u001b[?25hBuilding wheels for collected packages: lmdb\n", + " Building wheel for lmdb (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for lmdb: filename=lmdb-0.94-cp37-cp37m-linux_x86_64.whl size=219741 sha256=f3d72c383c674d7c07279fb6794cb9baf16a4cc2fb0bd255e67feae88fa149d4\n", + " Stored in directory: /root/.cache/pip/wheels/57/40/51/3fe10a4a559a91352579a27cbcca490f279bacb54209713c4b\n", + "Successfully built lmdb\n", + "Installing collected packages: lmdb\n", + " Found existing installation: lmdb 0.99\n", + " Uninstalling lmdb-0.99:\n", + " Successfully uninstalled lmdb-0.99\n", + "Successfully installed lmdb-0.94\n", + "Collecting tensorboardX==1.2\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/c5/22/43f4f0318f7c68a1000dbb700a353b745584bc2397437832d15ba69ea5f1/tensorboardX-1.2-py2.py3-none-any.whl (44kB)\n", + "\u001b[K |████████████████████████████████| 51kB 3.3MB/s \n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from tensorboardX==1.2) (1.16.4)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from tensorboardX==1.2) (1.15.0)\n", + "Requirement already satisfied: protobuf>=0.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorboardX==1.2) (3.12.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=0.3.2->tensorboardX==1.2) (56.0.0)\n", + "Installing collected packages: tensorboardX\n", + "Successfully installed tensorboardX-1.2\n", + "Collecting tensorflow==2.4.0\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/94/0a/012cc33c643d844433d13001dd1db179e7020b05ddbbd0a9dc86c38a8efa/tensorflow-2.4.0-cp37-cp37m-manylinux2010_x86_64.whl (394.7MB)\n", + "\u001b[K |████████████████████████████████| 394.7MB 37kB/s \n", + "\u001b[?25hRequirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.36.2)\n", + "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.6.3)\n", + "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.3.0)\n", + "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.3.3)\n", + "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.15.0)\n", + "Requirement already satisfied: grpcio~=1.32.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.32.0)\n", + "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.7.4.3)\n", + "Requirement already satisfied: h5py~=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (2.10.0)\n", + "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.12)\n", + "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.1.2)\n", + "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.2.0)\n", + "Collecting numpy~=1.19.2\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/08/d6/a6aaa29fea945bc6c61d11f6e0697b325ff7446de5ffd62c2fa02f627048/numpy-1.19.5-cp37-cp37m-manylinux2010_x86_64.whl (14.8MB)\n", + "\u001b[K |████████████████████████████████| 14.8MB 351kB/s \n", + "\u001b[?25hRequirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.12.4)\n", + "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.12.0)\n", + "Requirement already satisfied: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (2.4.1)\n", + "Requirement already satisfied: tensorflow-estimator<2.5.0,>=2.4.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (2.4.0)\n", + "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.1.0)\n", + "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.12.1)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.9.2->tensorflow==2.4.0) (56.0.0)\n", + "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (3.3.4)\n", + "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.8.0)\n", + "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (2.23.0)\n", + "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (0.4.4)\n", + "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.0.1)\n", + "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.28.1)\n", + "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.0) (3.10.1)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (1.24.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (2020.12.5)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (2.10)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (3.0.4)\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.0) (1.3.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (0.2.8)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (4.2.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (4.7.2)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.0) (3.4.1)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.0) (3.1.0)\n", + "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (0.4.8)\n", + "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n", + "Installing collected packages: numpy, tensorflow\n", + " Found existing installation: numpy 1.16.4\n", + " Uninstalling numpy-1.16.4:\n", + " Successfully uninstalled numpy-1.16.4\n", + " Found existing installation: tensorflow 2.4.1\n", + " Uninstalling tensorflow-2.4.1:\n", + " Successfully uninstalled tensorflow-2.4.1\n", + "Successfully installed numpy-1.19.5 tensorflow-2.4.0\n", + "Collecting tensorpack==0.9.4\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/21/69/c825f9fbd60ebca7387ab81f5704bf43ba6605c452555e305cbe6fcca61d/tensorpack-0.9.4-py2.py3-none-any.whl (273kB)\n", + "\u001b[K |████████████████████████████████| 276kB 6.5MB/s \n", + "\u001b[?25hRequirement already satisfied: msgpack>=0.5.2 in /usr/local/lib/python3.7/dist-packages (from tensorpack==0.9.4) (1.0.2)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from tensorpack==0.9.4) (1.15.0)\n", + "Collecting msgpack-numpy>=0.4.4.2\n", + " Downloading https://files.pythonhosted.org/packages/19/05/05b8d7c69c6abb36a34325cc3150089bdafc359f0a81fb998d93c5d5c737/msgpack_numpy-0.4.7.1-py2.py3-none-any.whl\n", + "Requirement already satisfied: termcolor>=1.1 in /usr/local/lib/python3.7/dist-packages (from tensorpack==0.9.4) (1.1.0)\n", + "Requirement already satisfied: numpy>=1.14 in /usr/local/lib/python3.7/dist-packages (from tensorpack==0.9.4) (1.19.5)\n", + "Requirement already satisfied: tqdm>4.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorpack==0.9.4) (4.41.1)\n", + "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.7/dist-packages (from tensorpack==0.9.4) (0.8.9)\n", + "Requirement already satisfied: pyzmq>=16 in /usr/local/lib/python3.7/dist-packages (from tensorpack==0.9.4) (22.0.3)\n", + "Installing collected packages: msgpack-numpy, tensorpack\n", + "Successfully installed msgpack-numpy-0.4.7.1 tensorpack-0.9.4\n", + "Collecting tqdm==4.31.1\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/6c/4b/c38b5144cf167c4f52288517436ccafefe9dc01b8d1c190e18a6b154cd4a/tqdm-4.31.1-py2.py3-none-any.whl (48kB)\n", + "\u001b[K |████████████████████████████████| 51kB 3.3MB/s \n", + "\u001b[31mERROR: spacy 2.2.4 has requirement tqdm<5.0.0,>=4.38.0, but you'll have tqdm 4.31.1 which is incompatible.\u001b[0m\n", + "\u001b[31mERROR: fbprophet 0.7.1 has requirement tqdm>=4.36.1, but you'll have tqdm 4.31.1 which is incompatible.\u001b[0m\n", + "\u001b[?25hInstalling collected packages: tqdm\n", + " Found existing installation: tqdm 4.41.1\n", + " Uninstalling tqdm-4.41.1:\n", + " Successfully uninstalled tqdm-4.41.1\n", + "Successfully installed tqdm-4.31.1\n", + "Requirement already satisfied: easydict==1.9 in /usr/local/lib/python3.7/dist-packages (1.9)\n", + "Collecting PyYAML==5.1.2\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/e3/e8/b3212641ee2718d556df0f23f78de8303f068fe29cdaa7a91018849582fe/PyYAML-5.1.2.tar.gz (265kB)\n", + "\u001b[K |████████████████████████████████| 266kB 4.9MB/s \n", + "\u001b[?25hBuilding wheels for collected packages: PyYAML\n", + " Building wheel for PyYAML (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for PyYAML: filename=PyYAML-5.1.2-cp37-cp37m-linux_x86_64.whl size=44103 sha256=f4c1a67893c8e98785a98e43f83e404a4cdb634c49f4cd7a4412ea70bf2904f1\n", + " Stored in directory: /root/.cache/pip/wheels/d9/45/dd/65f0b38450c47cf7e5312883deb97d065e030c5cca0a365030\n", + "Successfully built PyYAML\n", + "Installing collected packages: PyYAML\n", + " Found existing installation: PyYAML 3.13\n", + " Uninstalling PyYAML-3.13:\n", + " Successfully uninstalled PyYAML-3.13\n", + "Successfully installed PyYAML-5.1.2\n", + "Collecting jsonlines==1.2.0\n", + " Downloading https://files.pythonhosted.org/packages/4f/9a/ab96291470e305504aa4b7a2e0ec132e930da89eb3ca7a82fbe03167c131/jsonlines-1.2.0-py2.py3-none-any.whl\n", + "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from jsonlines==1.2.0) (1.15.0)\n", + "Installing collected packages: jsonlines\n", + "Successfully installed jsonlines-1.2.0\n", + "Collecting json-lines==0.5.0\n", + " Downloading https://files.pythonhosted.org/packages/7f/0f/79c96c0d26b276c583484fe8209e5ebbb416a920309568650325f6e1de73/json_lines-0.5.0-py2.py3-none-any.whl\n", + "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from json-lines==0.5.0) (1.15.0)\n", + "Installing collected packages: json-lines\n", + "Successfully installed json-lines-0.5.0\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (3.2.2)\n", + "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.19.5)\n", + "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.4.7)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.3.1)\n", + "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.8.1)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (0.10.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib) (1.15.0)\n", + "Requirement already satisfied: Cython in /usr/local/lib/python3.7/dist-packages (0.29.22)\n", + "Collecting python-prctl\n", + " Downloading https://files.pythonhosted.org/packages/c0/99/be5393cfe9c16376b4f515d90a68b11f1840143ac1890e9008bc176cf6a6/python-prctl-1.8.1.tar.gz\n", + "\u001b[31mERROR: Command errored out with exit status 1: python setup.py egg_info Check the logs for full command output.\u001b[0m\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.7/dist-packages (1.0.2)\n", + "Requirement already satisfied: msgpack-numpy in /usr/local/lib/python3.7/dist-packages (0.4.7.1)\n", + "Requirement already satisfied: msgpack>=0.5.2 in /usr/local/lib/python3.7/dist-packages (from msgpack-numpy) (1.0.2)\n", + "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from msgpack-numpy) (1.19.5)\n", + "Collecting opencv-python==4.2.0.34\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/d0/f0/cfe88d262c67825b20d396c778beca21829da061717c7aaa8b421ae5132e/opencv_python-4.2.0.34-cp37-cp37m-manylinux1_x86_64.whl (28.2MB)\n", + "\u001b[K |████████████████████████████████| 28.2MB 211kB/s \n", + "\u001b[?25hRequirement already satisfied: numpy>=1.14.5 in /usr/local/lib/python3.7/dist-packages (from opencv-python==4.2.0.34) (1.19.5)\n", + "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n", + "Installing collected packages: opencv-python\n", + " Found existing installation: opencv-python 4.1.2.30\n", + " Uninstalling opencv-python-4.1.2.30:\n", + " Successfully uninstalled opencv-python-4.1.2.30\n", + "Successfully installed opencv-python-4.2.0.34\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QuQKMp3GhSCa" + }, + "source": [ + "### install the module failed in requirements.txt" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "4Uj4965VPdR2", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "760de0cb-c341-41e3-9803-b72cc9938415" + }, + "source": [ + "!apt-get -qq install -y python-prctl\n", + "# apt-get, not pip, worked for this module" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Selecting previously unselected package python-prctl.\n", + "(Reading database ... 160690 files and directories currently installed.)\n", + "Preparing to unpack .../python-prctl_1.6.1-2build1_amd64.deb ...\n", + "Unpacking python-prctl (1.6.1-2build1) ...\n", + "Setting up python-prctl (1.6.1-2build1) ...\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ndqkGgTyR6wZ" + }, + "source": [ + "### apex\n", + "- https://github.com/NVIDIA/apex" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "CBZhkh8OR3IR", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "outputId": "a63f21c3-f7a3-41c8-83da-9043e7b4da88" + }, + "source": [ + "# https://stackoverflow.com/questions/57284345/how-to-install-nvidia-apex-on-google-colab\n", + "!git clone https://github.com/NVIDIA/apex\n", + "%cd apex\n", + "# !pip install -v --disable-pip-version-check --no-cache-dir ./\n", + "# https://github.com/NVIDIA/apex/issues/161\n", + "!pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\n", + "%cd ../\n", + "%pwd" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Cloning into 'apex'...\n", + "remote: Enumerating objects: 8038, done.\u001b[K\n", + "remote: Counting objects: 100% (125/125), done.\u001b[K\n", + "remote: Compressing objects: 100% (91/91), done.\u001b[K\n", + "remote: Total 8038 (delta 58), reused 68 (delta 29), pack-reused 7913\u001b[K\n", + "Receiving objects: 100% (8038/8038), 14.10 MiB | 28.88 MiB/s, done.\n", + "Resolving deltas: 100% (5459/5459), done.\n", + "/content/vilbert-multi-task/apex\n", + "/usr/local/lib/python3.7/dist-packages/pip/_internal/commands/install.py:283: UserWarning: Disabling all use of wheels due to the use of --build-options / --global-options / --install-options.\n", + " cmdoptions.check_install_build_global(options)\n", + "Created temporary directory: /tmp/pip-ephem-wheel-cache-20agkckj\n", + "Created temporary directory: /tmp/pip-req-tracker-6hkx_oj4\n", + "Created requirements tracker '/tmp/pip-req-tracker-6hkx_oj4'\n", + "Created temporary directory: /tmp/pip-install-lcw1rcqy\n", + "Processing /content/vilbert-multi-task/apex\n", + " Created temporary directory: /tmp/pip-req-build-_3dx09ry\n", + " Added file:///content/vilbert-multi-task/apex to build tracker '/tmp/pip-req-tracker-6hkx_oj4'\n", + " Running setup.py (path:/tmp/pip-req-build-_3dx09ry/setup.py) egg_info for package from file:///content/vilbert-multi-task/apex\n", + " Running command python setup.py egg_info\n", + "\n", + "\n", + " torch.__version__ = 1.8.1+cu101\n", + "\n", + "\n", + " running egg_info\n", + " creating /tmp/pip-req-build-_3dx09ry/pip-egg-info/apex.egg-info\n", + " writing /tmp/pip-req-build-_3dx09ry/pip-egg-info/apex.egg-info/PKG-INFO\n", + " writing dependency_links to /tmp/pip-req-build-_3dx09ry/pip-egg-info/apex.egg-info/dependency_links.txt\n", + " writing top-level names to /tmp/pip-req-build-_3dx09ry/pip-egg-info/apex.egg-info/top_level.txt\n", + " writing manifest file '/tmp/pip-req-build-_3dx09ry/pip-egg-info/apex.egg-info/SOURCES.txt'\n", + " adding license file 'LICENSE' (matched pattern 'LICEN[CS]E*')\n", + " writing manifest file '/tmp/pip-req-build-_3dx09ry/pip-egg-info/apex.egg-info/SOURCES.txt'\n", + " /tmp/pip-req-build-_3dx09ry/setup.py:67: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n", + " warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n", + " Source in /tmp/pip-req-build-_3dx09ry has version 0.1, which satisfies requirement apex==0.1 from file:///content/vilbert-multi-task/apex\n", + " Removed apex==0.1 from file:///content/vilbert-multi-task/apex from build tracker '/tmp/pip-req-tracker-6hkx_oj4'\n", + "Skipping wheel build for apex, due to binaries being disabled for it.\n", + "Installing collected packages: apex\n", + " Created temporary directory: /tmp/pip-record-shvsh4gq\n", + " Running command /usr/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-req-build-_3dx09ry/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-req-build-_3dx09ry/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' --cpp_ext --cuda_ext install --record /tmp/pip-record-shvsh4gq/install-record.txt --single-version-externally-managed --compile\n", + "\n", + "\n", + " torch.__version__ = 1.8.1+cu101\n", + "\n", + "\n", + " /tmp/pip-req-build-_3dx09ry/setup.py:67: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n", + " warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n", + "\n", + " Compiling cuda extensions with\n", + " nvcc: NVIDIA (R) Cuda compiler driver\n", + " Copyright (c) 2005-2020 NVIDIA Corporation\n", + " Built on Wed_Jul_22_19:09:09_PDT_2020\n", + " Cuda compilation tools, release 11.0, V11.0.221\n", + " Build cuda_11.0_bu.TC445_37.28845127_0\n", + " from /usr/local/cuda/bin\n", + "\n", + " Traceback (most recent call last):\n", + " File \"\", line 1, in \n", + " File \"/tmp/pip-req-build-_3dx09ry/setup.py\", line 171, in \n", + " check_cuda_torch_binary_vs_bare_metal(torch.utils.cpp_extension.CUDA_HOME)\n", + " File \"/tmp/pip-req-build-_3dx09ry/setup.py\", line 106, in check_cuda_torch_binary_vs_bare_metal\n", + " \"https://github.com/NVIDIA/apex/pull/323#discussion_r287021798. \"\n", + " RuntimeError: Cuda extensions are being compiled with a version of Cuda that does not match the version used to compile Pytorch binaries. Pytorch binaries were compiled with Cuda 10.1.\n", + " In some cases, a minor-version mismatch will not cause later errors: https://github.com/NVIDIA/apex/pull/323#discussion_r287021798. You can try commenting out this check (at your own risk).\n", + " Running setup.py install for apex ... \u001b[?25l\u001b[?25herror\n", + "Cleaning up...\n", + " Removing source in /tmp/pip-req-build-_3dx09ry\n", + "Removed build tracker '/tmp/pip-req-tracker-6hkx_oj4'\n", + "\u001b[31mERROR: Command errored out with exit status 1: /usr/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-req-build-_3dx09ry/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-req-build-_3dx09ry/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' --cpp_ext --cuda_ext install --record /tmp/pip-record-shvsh4gq/install-record.txt --single-version-externally-managed --compile Check the logs for full command output.\u001b[0m\n", + "Exception information:\n", + "Traceback (most recent call last):\n", + " File \"/usr/local/lib/python3.7/dist-packages/pip/_internal/cli/base_command.py\", line 153, in _main\n", + " status = self.run(options, args)\n", + " File \"/usr/local/lib/python3.7/dist-packages/pip/_internal/commands/install.py\", line 455, in run\n", + " use_user_site=options.use_user_site,\n", + " File \"/usr/local/lib/python3.7/dist-packages/pip/_internal/req/__init__.py\", line 62, in install_given_reqs\n", + " **kwargs\n", + " File \"/usr/local/lib/python3.7/dist-packages/pip/_internal/req/req_install.py\", line 888, in install\n", + " cwd=self.unpacked_source_directory,\n", + " File \"/usr/local/lib/python3.7/dist-packages/pip/_internal/utils/subprocess.py\", line 275, in runner\n", + " spinner=spinner,\n", + " File \"/usr/local/lib/python3.7/dist-packages/pip/_internal/utils/subprocess.py\", line 242, in call_subprocess\n", + " raise InstallationError(exc_msg)\n", + "pip._internal.exceptions.InstallationError: Command errored out with exit status 1: /usr/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-req-build-_3dx09ry/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-req-build-_3dx09ry/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' --cpp_ext --cuda_ext install --record /tmp/pip-record-shvsh4gq/install-record.txt --single-version-externally-managed --compile Check the logs for full command output.\n", + "/content/vilbert-multi-task\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'/content/vilbert-multi-task'" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 3 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2lNfrmdmTsn1" + }, + "source": [ + "### setup.py" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "m4je-VOhTaEh", + "outputId": "b062b390-e89b-4832-aabb-154947f76266" + }, + "source": [ + "!python setup.py develop" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "running develop\n", + "running egg_info\n", + "creating vilbert_multi_task.egg-info\n", + "writing vilbert_multi_task.egg-info/PKG-INFO\n", + "writing dependency_links to vilbert_multi_task.egg-info/dependency_links.txt\n", + "writing top-level names to vilbert_multi_task.egg-info/top_level.txt\n", + "writing manifest file 'vilbert_multi_task.egg-info/SOURCES.txt'\n", + "adding license file 'LICENSE' (matched pattern 'LICEN[CS]E*')\n", + "writing manifest file 'vilbert_multi_task.egg-info/SOURCES.txt'\n", + "running build_ext\n", + "Creating /usr/local/lib/python3.7/dist-packages/vilbert-multi-task.egg-link (link to .)\n", + "Adding vilbert-multi-task 0.1.0 to easy-install.pth file\n", + "\n", + "Installed /content/vilbert-multi-task\n", + "Processing dependencies for vilbert-multi-task==0.1.0\n", + "Finished processing dependencies for vilbert-multi-task==0.1.0\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3HI14_EDWPJG" + }, + "source": [ + "### error handling" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "xCb1H1suWU1p", + "outputId": "2a475f24-b590-4bdf-ec69-f66d6e120d5d" + }, + "source": [ + "# TypeError: Couldn't build proto file into descriptor pool!\n", + "## https://github.com/ValvePython/csgo/issues/8\n", + "!pip uninstall -y protobuf\n", + "!pip install --no-binary=protobuf protobuf" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Uninstalling protobuf-3.12.4:\n", + " Successfully uninstalled protobuf-3.12.4\n", + "Collecting protobuf\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/48/c8/e90238d5c0de6808da7b2529f4b2bd66c59ee73caabdd9d5bc351512f8b6/protobuf-3.15.8.tar.gz (228kB)\n", + "\u001b[K |████████████████████████████████| 235kB 6.4MB/s \n", + "\u001b[?25hRequirement already satisfied: six>=1.9 in /usr/local/lib/python3.7/dist-packages (from protobuf) (1.15.0)\n", + "Skipping wheel build for protobuf, due to binaries being disabled for it.\n", + "Installing collected packages: protobuf\n", + " Running setup.py install for protobuf ... \u001b[?25l\u001b[?25hdone\n", + "Successfully installed protobuf-3.15.8\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Uko4GMXDXAGv", + "outputId": "55b5eb0c-cdb2-4ca2-c1b7-1bf4dd668e57" + }, + "source": [ + "# TypeError: Conflict register for file \"tensorboard/compat/proto/tensor_shape.proto\": tensorboard.TensorShapeProto is already defined in file \"tensorboardX/src/tensor_shape.proto\".\n", + "## xx\n", + "# !pip install allennlp\n", + "# !pip install allennlp-models\n", + "!pip install tensorboardX==1.8" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting tensorboardX==1.8\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/c3/12/dcaf67e1312475b26db9e45e7bb6f32b540671a9ee120b3a72d9e09bc517/tensorboardX-1.8-py2.py3-none-any.whl (216kB)\n", + "\r\u001b[K |█▌ | 10kB 22.2MB/s eta 0:00:01\r\u001b[K |███ | 20kB 15.9MB/s eta 0:00:01\r\u001b[K |████▌ | 30kB 9.7MB/s eta 0:00:01\r\u001b[K |██████ | 40kB 8.0MB/s eta 0:00:01\r\u001b[K |███████▋ | 51kB 5.4MB/s eta 0:00:01\r\u001b[K |█████████ | 61kB 5.9MB/s eta 0:00:01\r\u001b[K |██████████▋ | 71kB 6.2MB/s eta 0:00:01\r\u001b[K |████████████▏ | 81kB 6.6MB/s eta 0:00:01\r\u001b[K |█████████████▋ | 92kB 6.3MB/s eta 0:00:01\r\u001b[K |███████████████▏ | 102kB 6.5MB/s eta 0:00:01\r\u001b[K |████████████████▊ | 112kB 6.5MB/s eta 0:00:01\r\u001b[K |██████████████████▏ | 122kB 6.5MB/s eta 0:00:01\r\u001b[K |███████████████████▊ | 133kB 6.5MB/s eta 0:00:01\r\u001b[K |█████████████████████▏ | 143kB 6.5MB/s eta 0:00:01\r\u001b[K |██████████████████████▊ | 153kB 6.5MB/s eta 0:00:01\r\u001b[K |████████████████████████▎ | 163kB 6.5MB/s eta 0:00:01\r\u001b[K |█████████████████████████▊ | 174kB 6.5MB/s eta 0:00:01\r\u001b[K |███████████████████████████▎ | 184kB 6.5MB/s eta 0:00:01\r\u001b[K |████████████████████████████▉ | 194kB 6.5MB/s eta 0:00:01\r\u001b[K |██████████████████████████████▎ | 204kB 6.5MB/s eta 0:00:01\r\u001b[K |███████████████████████████████▉| 215kB 6.5MB/s eta 0:00:01\r\u001b[K |████████████████████████████████| 225kB 6.5MB/s \n", + "\u001b[?25hRequirement already satisfied: protobuf>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX==1.8) (3.15.8)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from tensorboardX==1.8) (1.19.5)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from tensorboardX==1.8) (1.15.0)\n", + "Installing collected packages: tensorboardX\n", + " Found existing installation: tensorboardX 1.2\n", + " Uninstalling tensorboardX-1.2:\n", + " Successfully uninstalled tensorboardX-1.2\n", + "Successfully installed tensorboardX-1.8\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "3oEGQeZZer9H", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "outputId": "97f7e36e-2ac3-4d60-e559-ba21a2e49024" + }, + "source": [ + "# File \"/content/vilbert-multi-task/tools/refer/refer.py\", line 49\n", + "# print 'loading dataset %s into memory...' % dataset ^\n", + "# SyntaxError: Missing parentheses in call to 'print'. Did you mean print('loading dataset %s into memory...' % dataset)?\n", + "## So they're using python2 script :(\n", + "### https://github.com/facebookresearch/vilbert-multi-task/issues/33\n", + "%rm -r tools/refer\n", + "%mkdir tools/refer\n", + "!git clone -b python3 https://github.com/lichengunc/refer tools/refer\n", + "%cd tools/refer\n", + "!python setup.py install\n", + "!make\n", + "%cd ../\n", + "%cd ../\n", + "%pwd" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Cloning into 'tools/refer'...\n", + "remote: Enumerating objects: 185, done.\u001b[K\n", + "remote: Counting objects: 100% (3/3), done.\u001b[K\n", + "remote: Compressing objects: 100% (3/3), done.\u001b[K\n", + "remote: Total 185 (delta 0), reused 0 (delta 0), pack-reused 182\u001b[K\n", + "Receiving objects: 100% (185/185), 71.66 MiB | 18.32 MiB/s, done.\n", + "Resolving deltas: 100% (72/72), done.\n", + "/content/vilbert-multi-task/tools/refer\n", + "Compiling external/_mask.pyx because it changed.\n", + "[1/1] Cythonizing external/_mask.pyx\n", + "/usr/local/lib/python3.7/dist-packages/Cython/Compiler/Main.py:369: FutureWarning: Cython directive 'language_level' not set, using 2 for now (Py2). This will change in a later release! File: /content/vilbert-multi-task/tools/refer/external/_mask.pyx\n", + " tree = Parsing.p_module(s, pxd, full_module_name)\n", + "running install\n", + "running build\n", + "running build_py\n", + "creating build\n", + "creating build/lib.linux-x86_64-3.7\n", + "creating build/lib.linux-x86_64-3.7/external\n", + "copying external/mask.py -> build/lib.linux-x86_64-3.7/external\n", + "copying external/__init__.py -> build/lib.linux-x86_64-3.7/external\n", + "running build_ext\n", + "building 'external._mask' extension\n", + "creating build/temp.linux-x86_64-3.7\n", + "creating build/temp.linux-x86_64-3.7/external\n", + "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -Iexternal -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c external/_mask.c -o build/temp.linux-x86_64-3.7/external/_mask.o -Wno-cpp -Wno-unused-function -std=c99\n", + "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -Iexternal -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c external/maskApi.c -o build/temp.linux-x86_64-3.7/external/maskApi.o -Wno-cpp -Wno-unused-function -std=c99\n", + "\u001b[01m\u001b[Kexternal/maskApi.c:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[KrleDecode\u001b[m\u001b[K’:\n", + "\u001b[01m\u001b[Kexternal/maskApi.c:46:7:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n", + " \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K( k=0; k2) x+=(long) cnts[m-2]; cnts[m++]=(uint) x;\n", + " \u001b[01;35m\u001b[K^~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[Kexternal/maskApi.c:227:34:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kif\u001b[m\u001b[K’\n", + " if(m>2) x+=(long) cnts[m-2]; \u001b[01;36m\u001b[Kcnts\u001b[m\u001b[K[m++]=(uint) x;\n", + " \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n", + "x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/external/_mask.o build/temp.linux-x86_64-3.7/external/maskApi.o -o build/lib.linux-x86_64-3.7/external/_mask.cpython-37m-x86_64-linux-gnu.so\n", + "running install_lib\n", + "creating /usr/local/lib/python3.7/dist-packages/external\n", + "copying build/lib.linux-x86_64-3.7/external/mask.py -> /usr/local/lib/python3.7/dist-packages/external\n", + "copying build/lib.linux-x86_64-3.7/external/__init__.py -> /usr/local/lib/python3.7/dist-packages/external\n", + "copying build/lib.linux-x86_64-3.7/external/_mask.cpython-37m-x86_64-linux-gnu.so -> /usr/local/lib/python3.7/dist-packages/external\n", + "byte-compiling /usr/local/lib/python3.7/dist-packages/external/mask.py to mask.cpython-37.pyc\n", + "byte-compiling /usr/local/lib/python3.7/dist-packages/external/__init__.py to __init__.cpython-37.pyc\n", + "running install_egg_info\n", + "Writing /usr/local/lib/python3.7/dist-packages/external-2.0.egg-info\n", + "# install pycocotools/mask locally\n", + "# copy from https://github.com/pdollar/coco.git\n", + "python setup.py build_ext --inplace\n", + "running build_ext\n", + "building 'external._mask' extension\n", + "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -Iexternal -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c external/_mask.c -o build/temp.linux-x86_64-3.7/external/_mask.o -Wno-cpp -Wno-unused-function -std=c99\n", + "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -Iexternal -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c external/maskApi.c -o build/temp.linux-x86_64-3.7/external/maskApi.o -Wno-cpp -Wno-unused-function -std=c99\n", + "\u001b[01m\u001b[Kexternal/maskApi.c:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[KrleDecode\u001b[m\u001b[K’:\n", + "\u001b[01m\u001b[Kexternal/maskApi.c:46:7:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n", + " \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K( k=0; k2) x+=(long) cnts[m-2]; cnts[m++]=(uint) x;\n", + " \u001b[01;35m\u001b[K^~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[Kexternal/maskApi.c:227:34:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kif\u001b[m\u001b[K’\n", + " if(m>2) x+=(long) cnts[m-2]; \u001b[01;36m\u001b[Kcnts\u001b[m\u001b[K[m++]=(uint) x;\n", + " \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n", + "x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/external/_mask.o build/temp.linux-x86_64-3.7/external/maskApi.o -o /content/vilbert-multi-task/tools/refer/external/_mask.cpython-37m-x86_64-linux-gnu.so\n", + "rm -rf build\n", + "/content/vilbert-multi-task/tools\n", + "/content/vilbert-multi-task\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'/content/vilbert-multi-task'" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 7 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K_e6n1GnU5cS" + }, + "source": [ + "## download model\n", + "- https://stackoverflow.com/questions/49576657/is-there-anyway-i-can-download-the-file-in-google-colaboratory" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "--zTfu61U4n1", + "outputId": "6d35c615-9306-4078-9892-a55f1cb9751c" + }, + "source": [ + "!wget https://dl.fbaipublicfiles.com/vilbert-multi-task/multi_task_model.bin" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "--2021-04-28 09:06:33-- https://dl.fbaipublicfiles.com/vilbert-multi-task/multi_task_model.bin\n", + "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.74.142, 104.22.75.142, ...\n", + "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 1072254755 (1023M) [application/octet-stream]\n", + "Saving to: ‘multi_task_model.bin’\n", + "\n", + "multi_task_model.bi 100%[===================>] 1023M 32.8MB/s in 32s \n", + "\n", + "2021-04-28 09:07:06 (31.8 MB/s) - ‘multi_task_model.bin’ saved [1072254755/1072254755]\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LYEkc3kmmW2S" + }, + "source": [ + "## load data\n", + "- https://github.com/facebookresearch/vilbert-multi-task/tree/master/data\n", + " - In order not to exhaust the storage, extract VQA related files only" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "kwItao4-maiP", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "6eb50535-995d-4896-b198-c13763788f85" + }, + "source": [ + "# %cd data\n", + "!wget https://dl.fbaipublicfiles.com/vilbert-multi-task/datasets.tar.gz\n", + "# !tar tvf datasets.tar.gz\n", + "!tar xvf datasets.tar.gz $(tar tf datasets.tar.gz | grep 'VQA')\n", + "# !tar xvf datasets.tar.gz $(tar tf datasets.tar.gz | grep -i 'coco')\n", + "# !tar xf datasets.tar.gz --wildcards *coco\n", + "# !tar xf datasets.tar.gz" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "--2021-04-28 09:07:06-- https://dl.fbaipublicfiles.com/vilbert-multi-task/datasets.tar.gz\n", + "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.75.142, 104.22.74.142, ...\n", + "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 97480948100 (91G) [application/gzip]\n", + "Saving to: ‘datasets.tar.gz’\n", + "\n", + "datasets.tar.gz 26%[====> ] 24.15G 8.55MB/s in 25m 29s \n", + "\n", + "2021-04-28 09:32:36 (16.2 MB/s) - Connection closed at byte 25926144378. Retrying.\n", + "\n", + "--2021-04-28 09:32:37-- (try: 2) https://dl.fbaipublicfiles.com/vilbert-multi-task/datasets.tar.gz\n", + "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\n", + "HTTP request sent, awaiting response... 206 Partial Content\n", + "Length: 97480948100 (91G), 71554803722 (67G) remaining [application/gzip]\n", + "Saving to: ‘datasets.tar.gz’\n", + "\n", + "datasets.tar.gz 100%[+++++==============>] 90.79G 32.1MB/s in 36m 27s \n", + "\n", + "2021-04-28 10:09:05 (31.2 MB/s) - ‘datasets.tar.gz’ saved [97480948100/97480948100]\n", + "\n", + "datasets/VQA/\n", + "datasets/VQA/cache/\n", + "datasets/VQA/cache/train_target.pkl\n", + "datasets/VQA/cache/trainval_ans2label.pkl\n", + "datasets/VQA/cache/trainval_label2ans.pkl\n", + "datasets/VQA/cache/val_target.pkl\n", + "datasets/VQA/cache/coco_test_ids.npy\n", + "datasets/VQA/cache/VQA_trainval_23.pkl\n", + "datasets/VQA/cache/VQA_minval_23.pkl\n", + "datasets/VQA/cache/coco_test_ids_small.npy\n", + "datasets/VQA/cache/VQA_trainval_16.pkl\n", + "datasets/VQA/cache/VQA_minval_16.pkl\n", + "datasets/VQA/cache/VQA_test_23_cleaned.pkl\n", + "datasets/VQA/cache/VQA_trainval_23_cleaned.pkl\n", + "datasets/VQA/cache/VQA_minval_23_cleaned.pkl\n", + "datasets/VQA/cache/VQA_trainval_23_cleaned_final.pkl\n", + "datasets/VQA/cache/VQA_trainval_23_cleanedfinal.pkl\n", + "datasets/VQA/v2_mscoco_train2014_annotations.json\n", + "datasets/VQA/v2_mscoco_val2014_annotations.json\n", + "datasets/VQA/v2_OpenEnded_mscoco_test2015_questions.json\n", + "datasets/VQA/v2_OpenEnded_mscoco_test-dev2015_questions.json\n", + "datasets/VQA/v2_OpenEnded_mscoco_train2014_questions.json\n", + "datasets/VQA/v2_OpenEnded_mscoco_val2014_questions.json\n", + "tar: datasets/VQA/cache: Not found in archive\n", + "tar: datasets/VQA/cache/train_target.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/trainval_ans2label.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/trainval_label2ans.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/val_target.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/coco_test_ids.npy: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_trainval_23.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_minval_23.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/coco_test_ids_small.npy: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_trainval_16.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_minval_16.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_test_23_cleaned.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_trainval_23_cleaned.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_minval_23_cleaned.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_trainval_23_cleaned_final.pkl: Not found in archive\n", + "tar: datasets/VQA/cache/VQA_trainval_23_cleanedfinal.pkl: Not found in archive\n", + "tar: datasets/VQA/v2_mscoco_train2014_annotations.json: Not found in archive\n", + "tar: datasets/VQA/v2_mscoco_val2014_annotations.json: Not found in archive\n", + "tar: datasets/VQA/v2_OpenEnded_mscoco_test2015_questions.json: Not found in archive\n", + "tar: datasets/VQA/v2_OpenEnded_mscoco_test-dev2015_questions.json: Not found in archive\n", + "tar: datasets/VQA/v2_OpenEnded_mscoco_train2014_questions.json: Not found in archive\n", + "tar: datasets/VQA/v2_OpenEnded_mscoco_val2014_questions.json: Not found in archive\n", + "tar: Exiting with failure status due to previous errors\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RTeHEqhHn9ZO", + "outputId": "961f9e09-3e89-49b7-c897-20087a835977" + }, + "source": [ + "# %cd data/\n", + "%rm datasets.tar.gz\n", + "%cd /content/vilbert-multi-task/" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/content/vilbert-multi-task\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "I1VkgsohmmHV", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "10d4d80f-e44c-4c8c-88e2-4ecb5ece8e46" + }, + "source": [ + "# !mkdir datasets\n", + "%cd datasets\n", + "!mkdir coco\n", + "%cd coco\n", + "!mkdir features_100\n", + "%cd features_100\n", + "# !mkdir COCO_test_resnext152_faster_rcnn_genome.lmdb\n", + "!mkdir COCO_trainval_resnext152_faster_rcnn_genome.lmdb\n", + "!wget https://dl.fbaipublicfiles.com/vilbert-multi-task/datasets/coco/features_100/COCO_trainval_resnext152_faster_rcnn_genome.lmdb/data.mdb && mv data.mdb COCO_trainval_resnext152_faster_rcnn_genome.lmdb/\n", + "# !wget https://dl.fbaipublicfiles.com/vilbert-multi-task/datasets/coco/features_100/COCO_test_resnext152_faster_rcnn_genome.lmdb/data.mdb && mv data.mdb COCO_test_resnext152_faster_rcnn_genome.lmdb/\n", + "%cd /content/vilbert-multi-task/" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/content/vilbert-multi-task/datasets\n", + "/content/vilbert-multi-task/datasets/coco\n", + "/content/vilbert-multi-task/datasets/coco/features_100\n", + "--2021-04-28 11:26:22-- https://dl.fbaipublicfiles.com/vilbert-multi-task/datasets/coco/features_100/COCO_trainval_resnext152_faster_rcnn_genome.lmdb/data.mdb\n", + "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.75.142, 172.67.9.4, 104.22.74.142, ...\n", + "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.75.142|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 101507162112 (95G) [application/octet-stream]\n", + "Saving to: ‘data.mdb’\n", + "\n", + "data.mdb 100%[===================>] 94.54G 31.3MB/s in 68m 43s \n", + "\n", + "2021-04-28 12:35:05 (23.5 MB/s) - ‘data.mdb’ saved [101507162112/101507162112]\n", + "\n", + "/content/vilbert-multi-task\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oZaa0Gtl1jlC", + "outputId": "20931650-6ed6-450e-f1a9-7ef0627c178e" + }, + "source": [ + "%cd /content/vilbert-multi-task/" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/content/vilbert-multi-task\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FYUdvXo5cJd0" + }, + "source": [ + "## Run train_tasks.py as fine tuning\n", + "- VQA as classification problem\n", + " - https://github.com/facebookresearch/vilbert-multi-task/blob/master/vilbert_tasks.yml\n", + " - https://arxiv.org/abs/1707.07998" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "vLZX_7bTWvL1", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "160e00f5-4798-44f1-ffa0-1f20b4018413" + }, + "source": [ + "# Decrease batch size to avoid memory error\n", + "## https://datascience.stackexchange.com/questions/13669/how-to-export-one-cell-of-a-jupyter-notebook\n", + "%%writefile vilbert_tasks.yml\n", + "TASK1:\n", + " name: VQA\n", + " type: VL-classifier\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 1\n", + " dataroot: datasets/VQA/\n", + " features_h5path1: datasets/coco/features_100/COCO_trainval_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 23\n", + " max_region_num: 101\n", + " batch_size: 8\n", + " eval_batch_size: 8\n", + " train_split: trainval\n", + " val_split: minval\n", + " lr: 0.00004\n", + " num_epoch: 20\n", + "TASK2:\n", + " name: GenomeQA\n", + " type: VL-classifier\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 2\n", + " dataroot: datasets/visual_genome/\n", + " features_h5path1: datasets/visual_genome/vg_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 26\n", + " max_region_num: 101\n", + " batch_size: 128\n", + " eval_batch_size: 1024\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00004\n", + " num_epoch: 20\n", + "TASK3:\n", + " name: VisualDialog\n", + " type: VL-logit\n", + " loss: CrossEntropyLoss\n", + " process: dialog\n", + " task_id: 3\n", + " dataroot: datasets/visual_dialog\n", + " features_h5path1: datasets/coco/features_100/COCO_trainval_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: datasets/visual_dialog/visdial_1.0_train.json\n", + " val_annotations_jsonpath: datasets/visual_dialog/visdial_1.0_val.json\n", + " max_seq_length: 16\n", + " max_region_num: 101\n", + " batch_size: 64\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00004\n", + " num_epoch: 20\n", + "TASK4:\n", + " name: Visual7w\n", + " type: V-logit-mc\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 4\n", + " dataroot: datasets/visual7w\n", + " features_h5path1: datasets/visual7w/visual7w_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: datasets/visual7w/visual7w_gt_resnext152_faster_rcnn_genome.lmdb\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 20\n", + " max_region_num: 200\n", + " batch_size: 256\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK5:\n", + " name: VCR_Q-A\n", + " type: VL-logit\n", + " loss: CrossEntropyLoss\n", + " process: expand\n", + " task_id: 5\n", + " dataroot: datasets/VCR\n", + " features_h5path1: datasets/VCR/VCR_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: datasets/VCR/VCR_gt_resnext152_faster_rcnn_genome.lmdb\n", + " train_annotations_jsonpath: datasets/VCR/train.jsonl\n", + " val_annotations_jsonpath: datasets/VCR/val.jsonl\n", + " max_seq_length: 60\n", + " max_region_num: 101\n", + " batch_size: 64\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00002\n", + " num_epoch: 10\n", + "TASK6:\n", + " name: VCR_QA-R\n", + " type: VL-logit\n", + " loss: CrossEntropyLoss\n", + " process: expand\n", + " task_id: 6\n", + " dataroot: datasets/VCR\n", + " features_h5path1: datasets/VCR/VCR_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: datasets/VCR/VCR_gt_resnext152_faster_rcnn_genome.lmdb\n", + " train_annotations_jsonpath: datasets/VCR/train.jsonl\n", + " val_annotations_jsonpath: datasets/VCR/val.jsonl\n", + " max_seq_length: 80\n", + " max_region_num: 101\n", + " batch_size: 64\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00002\n", + " num_epoch: 10\n", + "TASK7:\n", + " name: RetrievalCOCO\n", + " type: VL-logit\n", + " loss: CrossEntropyLoss\n", + " process: retrieval\n", + " task_id: 7\n", + " dataroot: datasets/cocoRetreival\n", + " features_h5path1: datasets/coco/features_100/COCO_trainval_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: datasets/cocoRetreival/all_data_final_train_2014.jsonline\n", + " val_annotations_jsonpath: datasets/cocoRetreival/all_data_final_test_set0_2014.jsonline\n", + " max_seq_length: 30\n", + " max_region_num: 101\n", + " batch_size: 128\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK8:\n", + " name: RetrievalFlickr30k\n", + " type: VL-logit\n", + " loss: CrossEntropyLoss\n", + " process: retrieval\n", + " task_id: 8\n", + " dataroot: datasets/flickr30k\n", + " features_h5path1: datasets/flickr30k/flickr30k_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: datasets/flickr30k/all_data_final_train_2014.jsonline\n", + " val_annotations_jsonpath: datasets/flickr30k/all_data_final_test_set0_2014.jsonline\n", + " max_seq_length: 30\n", + " max_region_num: 101\n", + " batch_size: 128\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK9:\n", + " name: refcoco\n", + " type: V-logit\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 9\n", + " dataroot: datasets/refcoco\n", + " features_h5path1: datasets/refcoco/refcoco_unc/refcoco_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: datasets/refcoco/refcoco_unc/refcoco_gt_resnext152_faster_rcnn_genome.lmdb\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 20\n", + " max_region_num: 101\n", + " batch_size: 256\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK10:\n", + " name: refcoco+\n", + " type: V-logit\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 10\n", + " dataroot: datasets/refcoco\n", + " features_h5path1: datasets/refcoco/refcoco+_unc/refcoco+_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: datasets/refcoco/refcoco+_unc/refcoco+_gt_resnext152_faster_rcnn_genome.lmdb\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 20\n", + " max_region_num: 101\n", + " batch_size: 256\n", + " eval_batch_size: 1024\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK11:\n", + " name: refcocog\n", + " type: V-logit\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 11\n", + " dataroot: datasets/refcoco\n", + " features_h5path1: datasets/refcoco/refcocog_umd/refcocog_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: datasets/refcoco/refcocog_umd/refcocog_gt_resnext152_faster_rcnn_genome.lmdb\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 20\n", + " max_region_num: 101\n", + " batch_size: 256\n", + " train_split: train\n", + " val_split: val\n", + " start_iteration: 0\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK12:\n", + " name: NLVR2\n", + " type: VL-binary-classifier\n", + " loss: BCEWithLogitLoss\n", + " process: nlvr\n", + " task_id: 12\n", + " dataroot: datasets/nlvr2/\n", + " features_h5path1: datasets/nlvr2/nlvr2_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 40\n", + " max_region_num: 101\n", + " batch_size: 64\n", + " eval_batch_size: 512\n", + " train_split: train\n", + " val_split: dev\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK13: \n", + " name: VisualEntailment\n", + " type: VL-tri-classifier\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 13\n", + " dataroot: datasets/visual_entailment/\n", + " features_h5path1: datasets/flickr30k/flickr30k_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 56\n", + " max_region_num: 101\n", + " batch_size: 256\n", + " eval_batch_size: 1024\n", + " train_split: train\n", + " val_split: dev\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK14:\n", + " name: GuessWhat\n", + " type: VL-tri-classifier\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 14\n", + " dataroot: datasets/guesswhat/\n", + " features_h5path1: datasets/coco/features_100/COCO_trainval_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 25\n", + " max_region_num: 101\n", + " batch_size: 256\n", + " eval_batch_size: 1024\n", + " train_split: train\n", + " val_split: valid\n", + " lr: 0.00004\n", + " num_epoch: 20\n", + "TASK15:\n", + " name: GQA\n", + " type: VL-classifier-GQA\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 15\n", + " dataroot: datasets/gqa/\n", + " features_h5path1: datasets/gqa/gqa_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 26\n", + " max_region_num: 101\n", + " batch_size: 128\n", + " eval_batch_size: 1024\n", + " train_split: trainval\n", + " val_split: minval\n", + " lr: 0.00004\n", + " num_epoch: 20\n", + "TASK16:\n", + " name: Foil\n", + " type: VL-binary-classifier\n", + " loss: CrossEntropyLoss\n", + " process: normal\n", + " task_id: 16\n", + " dataroot: datasets/Foil\n", + " features_h5path1: datasets/coco/features_100/COCO_trainval_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: ''\n", + " train_annotations_jsonpath: datasets/Foil/foilv1.0_train_2017.json\n", + " val_annotations_jsonpath: datasets/Foil/foilv1.0_test_2017.json\n", + " max_seq_length: 20\n", + " max_region_num: 101\n", + " batch_size: 256\n", + " eval_batch_size: 1024\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.00004\n", + " num_epoch: 20\n", + "TASK17:\n", + " name: GuessWhatPointing\n", + " type: V-logit-mc\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 17\n", + " dataroot: datasets/guesswhat/\n", + " features_h5path1: datasets/coco/features_100/COCO_trainval_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: datasets/guesswhat/guesswhat_gt_resnext152_faster_rcnn_genome.lmdb\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 256\n", + " max_region_num: 306\n", + " batch_size: 64\n", + " train_split: train\n", + " val_split: valid\n", + " lr: 0.00002\n", + " num_epoch: 20\n", + "TASK18:\n", + " name: FlickrGrounding\n", + " type: V-logit\n", + " loss: BCEWithLogitLoss\n", + " process: normal\n", + " task_id: 18\n", + " dataroot: datasets/flickr30k/\n", + " features_h5path1: datasets/flickr30k/flickr30k_resnext152_faster_rcnn_genome.lmdb\n", + " features_h5path2: datasets/flickr30k/flickr30k_gt_resnext152_faster_rcnn_genome.lmdb\n", + " train_annotations_jsonpath: ''\n", + " val_annotations_jsonpath: ''\n", + " max_seq_length: 24\n", + " max_region_num: 200\n", + " batch_size: 256\n", + " train_split: train\n", + " val_split: val\n", + " lr: 0.000002\n", + " num_epoch: 20" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Overwriting vilbert_tasks.yml\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RXlUoEu7cIHH", + "outputId": "e52129c7-993b-4cdd-b871-f7c5481e115d" + }, + "source": [ + "!python train_tasks.py --bert_model bert-base-uncased --from_pretrained multi_task_model.bin --config_file config/bert_base_6layer_6conect.json --tasks 1 --lr_scheduler 'warmup_linear' --train_iter_gap 4 --task_specific_tokens --save_name finetune_from_multi_task_model" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "2021-04-28 12:35:12.334426: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n", + "04/28/2021 12:35:17 - INFO - vilbert.vilbert - Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n", + "04/28/2021 12:35:17 - INFO - __main__ - device: cuda n_gpu: 1, distributed training: False, 16-bits training: False\n", + "04/28/2021 12:35:18 - INFO - pytorch_transformers.file_utils - https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt not found in cache, downloading to /tmp/tmp7k414p6j\n", + "100% 231508/231508 [00:00<00:00, 2628790.99B/s]\n", + "04/28/2021 12:35:18 - INFO - pytorch_transformers.file_utils - copying /tmp/tmp7k414p6j to cache at /root/.cache/torch/pytorch_transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n", + "04/28/2021 12:35:18 - INFO - pytorch_transformers.file_utils - creating metadata file for /root/.cache/torch/pytorch_transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n", + "04/28/2021 12:35:18 - INFO - pytorch_transformers.file_utils - removing temp file /tmp/tmp7k414p6j\n", + "04/28/2021 12:35:18 - INFO - pytorch_transformers.tokenization_utils - loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /root/.cache/torch/pytorch_transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n", + "04/28/2021 12:35:18 - INFO - vilbert.task_utils - Loading VQA Dataset with batch size 8\n", + "04/28/2021 12:35:18 - INFO - vilbert.datasets.vqa_dataset - Loading from datasets/VQA/cache/VQA_trainval_23_cleaned.pkl\n", + "04/28/2021 12:37:13 - INFO - vilbert.datasets.vqa_dataset - Loading from datasets/VQA/cache/VQA_minval_23_cleaned.pkl\n", + "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:477: UserWarning: This DataLoader will create 16 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", + " cpuset_checked))\n", + "04/28/2021 12:37:14 - INFO - vilbert.utils - logging file at: save/VQA_bert_base_6layer_6conect-finetune_from_multi_task_model/logs\n", + "04/28/2021 12:37:14 - INFO - vilbert.utils - loading weights file multi_task_model.bin\n", + "559 559\n", + "***** Running training *****\n", + " Num Iters: {'TASK1': 67763}\n", + " Batch size: {'TASK1': 8}\n", + " Num steps: 1355260\n", + "Epoch: 0% 0/20 [00:00