From b5e34023f10ee8b5ec884a7bbefffb62c8449045 Mon Sep 17 00:00:00 2001 From: Jihyeong Lee Date: Fri, 28 Mar 2025 10:19:51 -0400 Subject: [PATCH 01/11] Added unsloth --- .gitignore | 6 + end2end/nano-gpt.sh | 0 end2end/unsloth_llama3/Gemma3_(4B).ipynb | 7431 ++++++++++++++++++++++ end2end/unsloth_llama3/llama3_1.py | 232 + microbench.sh | 4 +- 5 files changed, 7672 insertions(+), 1 deletion(-) mode change 100644 => 100755 end2end/nano-gpt.sh create mode 100644 end2end/unsloth_llama3/Gemma3_(4B).ipynb create mode 100644 end2end/unsloth_llama3/llama3_1.py mode change 100644 => 100755 microbench.sh diff --git a/.gitignore b/.gitignore index 82f9275..f7fcfb3 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,9 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + + +end2end/unsloth_llama3/outputs/* +end2end/unsloth_llama3/unsloth_compiled_cache/* +*.hatchet +*.nsys-rep diff --git a/end2end/nano-gpt.sh b/end2end/nano-gpt.sh old mode 100644 new mode 100755 diff --git a/end2end/unsloth_llama3/Gemma3_(4B).ipynb b/end2end/unsloth_llama3/Gemma3_(4B).ipynb new file mode 100644 index 0000000..820c1fd --- /dev/null +++ b/end2end/unsloth_llama3/Gemma3_(4B).ipynb @@ -0,0 +1,7431 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "NmbKJxjRjeWg" + }, + "source": [ + "To run this, press \"*Runtime*\" and press \"*Run all*\" on a **free** Tesla T4 Google Colab instance!\n", + "
\n", + "\n", + "\n", + " Join Discord if you need help + ⭐ Star us on Github ⭐\n", + "
\n", + "\n", + "To install Unsloth on your own computer, follow the installation instructions on our Github page [here](https://docs.unsloth.ai/get-started/installing-+-updating).\n", + "\n", + "You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & [how to save it](#Save)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-2D281t8jeWg" + }, + "source": [ + "### News" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SSY2XE1xjeWg" + }, + "source": [ + "**Read our [Gemma 3 blog](https://unsloth.ai/blog/gemma3) for what's new in Unsloth and our [Reasoning blog](https://unsloth.ai/blog/r1-reasoning) on how to train reasoning models.**\n", + "\n", + "Visit our docs for all our [model uploads](https://docs.unsloth.ai/get-started/all-our-models) and [notebooks](https://docs.unsloth.ai/get-started/unsloth-notebooks).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ySsPI_X1jeWh" + }, + "source": [ + "### Installation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "D_L_j6gejeWi" + }, + "outputs": [], + "source": [ + "%%capture\n", + "import os\n", + "if \"COLAB_\" not in \"\".join(os.environ.keys()):\n", + " !pip install unsloth vllm\n", + "else:\n", + " # [NOTE] Do the below ONLY in Colab! Use [[pip install unsloth vllm]]\n", + " !pip install --no-deps unsloth vllm\n", + "# Install latest Hugging Face for Gemma-3!\n", + "!pip install --no-deps git+https://github.com/huggingface/transformers@v4.49.0-Gemma-3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tZ3ezvsPjeWi" + }, + "outputs": [], + "source": [ + "#@title Colab Extra Install { display-mode: \"form\" }\n", + "%%capture\n", + "import os\n", + "if \"COLAB_\" not in \"\".join(os.environ.keys()):\n", + " !pip install unsloth vllm\n", + "else:\n", + " !pip install --no-deps unsloth vllm\n", + " # [NOTE] Do the below ONLY in Colab! Use [[pip install unsloth vllm]]\n", + " # Skip restarting message in Colab\n", + " import sys, re, requests; modules = list(sys.modules.keys())\n", + " for x in modules: sys.modules.pop(x) if \"PIL\" in x or \"google\" in x else None\n", + " !pip install --no-deps bitsandbytes accelerate xformers==0.0.29.post3 peft \"trl==0.15.2\" triton cut_cross_entropy unsloth_zoo\n", + " !pip install sentencepiece protobuf datasets huggingface_hub hf_transfer\n", + "\n", + " # vLLM requirements - vLLM breaks Colab due to reinstalling numpy\n", + " f = requests.get(\"https://raw.githubusercontent.com/vllm-project/vllm/refs/heads/main/requirements/common.txt\").content\n", + " with open(\"vllm_requirements.txt\", \"wb\") as file:\n", + " file.write(re.sub(rb\"(transformers|numpy|xformers)[^\\n]{1,}\\n\", b\"\", f))\n", + " !pip install -r vllm_requirements.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TGMWlrRdzwgf" + }, + "source": [ + "### Unsloth\n", + "\n", + "`FastModel` supports loading nearly any model now! This includes Vision and Text models!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 533, + "referenced_widgets": [ + "33815b4c0485402e838127b32ad14a15", + "81c3ee8d97d543ae83db3d2748ecd7cd", + "70f9bccb201a4ea79c1827057cd746f6", + "b32538030b8649e7851bfd58bef2786c", + "0abe1a2b12e54bb5a07db1a8f3a77738", + "e12bca7d49b149b5b1124c28a669db98", + "fab7a02d350946ae9563a05cfd04e22b", + "9be36a19ba86494389904b9fffdc2e48", + "899bf9a4dc594e178d0b95e3cbe08018", + "2fa2784330b2439ab884ce0966037961", + "51649c42cf2145f1b1e90c3436805350", + "eae020595f574192ad9d132853bbf6ec", + "17eaf723882e4efea38119978166fc75", + "409488926d2242c5a8e7b3d5b79c59db", + "26b61507603d453c8c24af24d301bdb9", + "c6c51a350a0b420aab957e3570098e18", + "cebd4fbf1fcf4ab2b65ecf539eda5a1e", + "f14ea72beac74152af5f970e634769ca", + "039f461e15214bd697501219bd9cbbd9", + "9f9d2f43fb5e47df883feb3126fe52e9", + "9e20d704e64a4aaabe5e495c468d9670", + "ae4e4537ed1d4df4b1677700d190a2d2", + "50881898da2f4b35a288ac9befe5024e", + "599375adc5f841d1864166e4d1bd617a", + "3bfc7d7dd81f49a59fc8ad0d6fff858b", + "a0c0025c82394e7fbc6d4cc9a9e9f72f", + "5f0a37b9edc74cbd822e2e71c6c8a956", + "a15e4524521b42108f49dda23ed56023", + "a4eaae1b30d442208257c1870d549738", + "84c68f2059f247628b672b1079130e9b", + "a88fccae11664baa82418c88639f3521", + "1ecb60e1d5934ea19a0d2c29aa01158d", + "f920f6cb263c45d59b851b7c6b631cb5", + "a37b4e454c6743a895f159f963366bc8", + "18fce8679d7d4961b88bb2162e7aa9eb", + "fd60bb21ba2e474cb4a6020bd302835e", + "0c3e3fbf02d84114906e939fcca108b5", + "ab71391446d8482c834429a937f7bb96", + "f9439c3c9b3b4c4a84ed67aa0601a530", + "a89575b4c58348ac98566c22ab7e4118", + "697a187a07204fdfb8556cf5c5028c6b", + "afdb7dfdc17548b39daea1f39d54b45c", + "c4a69698321d435c94211a9dee913c45", + "36c8135711194884be0e03cb5d3ae7e5", + "69a9d565de6a4d54b8f3989fa8b11941", + "090b146b8c6f4fa1ab0261d2a61be9de", + "d2df1617a4094dd295c49a0a72b269c3", + "54d963afc22a46ae93a6ca4bfaa18cf8", + "0b1768a5b5be4a4d9e14ed9e168044b4", + "708d05af00a64d19bb11dc839a5e68db", + "3d20e28d74e549c0a43686b214eebd87", + "dfac71c6372c46bfad46308bdb04480b", + "df50d8daa49a4955914704edb89baf61", + "52f091f99f0442f8bd14a50b7c870c1e", + "c859133fca324effb73ebc3520e746b6", + "f89c08592a25432497bb312f58a13c5c", + "c307400fb17e49ea9d835822e6e22633", + "57ed5097e05f4d92a5c492826f989123", + "db7e622bbd0f4357b5687a7c09c1f6fd", + "8864799f440c440c8ff8c0696e64215d", + "353fa47fb98c4070a150edec64503eaa", + "40bf6f1ffbc5479082fdd7ab153ea974", + "b5a06422fcac41eb97d2de95f14b1806", + "efa41d07d0fa4adda8025fe9490ed850", + "457c60d6a15d4314ba25d370be956a60", + "bc6f29c9a1e14ce8be374867b8be86ac", + "e003ee5cf1804ce3928b544b3fa7ba77", + "dd6e4f9b4c6d4260a62b920a6812fd07", + "4466f20e614a4cdabe1704859c2f1034", + "8949c35d68a043c5b1384774bb07b2ea", + "fca09f95775047efa9d481173f1ba261", + "879c6a0498e54e5c87145c7f7d32de7e", + "740d351b7de241a6acabf6c2853585b6", + "be2b1fb954444be089045a378673a958", + "9520485cc7c24c8584c3838717655012", + "94d9900bc8934de688e95e15c9d0c9bb", + "b469cdf580404f498feb062e4dbad10b", + "5975cf24b18e4082bd80e3f177e0ec15", + "b5b8482ef7c44e12a83795e7337521c2", + "ea9045a5c4504a5e96e6a7b13767fe4e", + "0cba80b626574c11a44c6ce09b5d6e80", + "f344c1ab154b4abdb84b2c221b6162a1", + "0881c055108340f7ab4b840ac1545cbb", + "77907c3444174858bbdc548dee8d0d37", + "a92be3fb752148d887e20afe300f9371", + "aa36bff36ae5448b892afea071fc1f1d", + "12d3049cca4a46c08cf5cdfcd5225248", + "6a9baf0a739c4790baf99b0ccadd6873", + "86c6d49a55b3477bbccc275dcb55fb52", + "77c5f8b431ba4c08b8f4d9d8f9fafc16", + "19c3ef35452d406cb18b72e38b631ee7", + "063db74d47814f95b560bd3bab11b55f", + "5c01ab4767104c0c96c42858317f8877", + "73a283e64f324c27b38216e683050b92", + "81bddfaa180d4875b5cdb5cc4ae45dab", + "77e843913b6e439ead9cf42725eedf3d", + "4f7e8b6b71484cce80ae9cdf0c481825", + "3530e2b431c041c6aeeaca4808ba0424", + "cc14e51320f34274ae12aa28b04183b7", + "f373f2c24f3b413aaa9fe1ccfb9c1eab", + "360142dac8c54a5eb902078ec42abb65", + "a4a64136f6fc48c799abf1701725534e", + "4c5e29de7224428bb87de46854ea915a", + "a3ce4f38be9a456c81146fba440c8e3f", + "70fdd31291b04dd68d66cf31c03d23ff", + "6feaf338d39440e78221649ac84af4a6", + "c9fbf40fa3dd4ba1b363802dc88764da", + "ae7f1fd06ddc4881934690675891855c", + "5e9ba3247edc4fafa7687338424ddccb", + "97d7e3420e24436cb351b1e9679ff8b6" + ] + }, + "id": "-Xbb0cuLzwgf", + "outputId": "3396a9bf-5d9b-45e7-a13e-ac57a78dd441" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "🦥 Unsloth Zoo will now patch everything to make training faster!\n", + "==((====))== Unsloth 2025.3.14: Fast Gemma3 patching. Transformers: 4.50.0.dev0.\n", + " \\\\ /| Tesla T4. Num GPUs = 1. Max memory: 14.741 GB. Platform: Linux.\n", + "O^O/ \\_/ \\ Torch: 2.6.0+cu124. CUDA: 7.5. CUDA Toolkit: 12.4. Triton: 3.2.0\n", + "\\ / Bfloat16 = FALSE. FA [Xformers = 0.0.29.post3. FA2 = False]\n", + " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", + "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n", + "Unsloth: Using float16 precision for gemma3 won't work! Using float32.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "33815b4c0485402e838127b32ad14a15", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "model.safetensors: 0%| | 0.00/4.44G [00:00\n", + "### Data Prep\n", + "We now use the `Gemma-3` format for conversation style finetunes. We use [Maxime Labonne's FineTome-100k](https://huggingface.co/datasets/mlabonne/FineTome-100k) dataset in ShareGPT style. Gemma-3 renders multi turn conversations like below:\n", + "\n", + "```\n", + "user\n", + "Hello!\n", + "model\n", + "Hey there!\n", + "```\n", + "\n", + "We use our `get_chat_template` function to get the correct chat template. We support `zephyr, chatml, mistral, llama, alpaca, vicuna, vicuna_old, phi3, llama3, phi4, qwen2.5, gemma3` and more." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LjY75GoYUCB8" + }, + "outputs": [], + "source": [ + "from unsloth.chat_templates import get_chat_template\n", + "tokenizer = get_chat_template(\n", + " tokenizer,\n", + " chat_template = \"gemma-3\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 113, + "referenced_widgets": [ + "55164fb3b49848beabfe0da534ab4b97", + "79c551f4fd374acdb112c3d8d82c58a0", + "be024d1de576448fb472c3349b2d6d9e", + "173033e49f9f46d9930c6afddc983509", + "2b95af79a12749549da9bd21f0a944db", + "006e1f7432f84b298f61bb597b8aab10", + "f496e27e3d6446c7997c6fc549de69fa", + "1cb8f93e225f427f92c7bf7aa99aba8c", + "89aa2b1f06ab47d7ac0e4cdfd70d7bfe", + "441ff92725ef40ce807daa1ff721faef", + "21136eec72034fcbb47a19f92aa0664d", + "a74d75b5fbe34eb7b1033d21777ecddb", + "3477f57107754fb58bf5080869c81092", + "95371925c6db4abca3d7987775d1e7b0", + "a431edec70ad43569fe75342678ad9d7", + "b2bf96a0eae14ea78c0530c2574a94c7", + "9511c493311b4d4e94d5dc0aca4eeffb", + "f4265abad437426b8efa9091110c77c9", + "be166331dbc5446c8a4f45e861183b71", + "45616dc4ade54cb28ae49c9489d810ca", + "7ff7c3b516d84706a47e91c64b50d171", + "7850eeab711f494399e82bf96f942888", + "2ef58068693547b8b3111d944703d188", + "ec4009400bb34f05bd1554db5bbd9ea8", + "018b82e39081491fb1137b8fc2707f03", + "e0387cc0b0d7426d9e6a7a1cd42bb760", + "3526caa6bc6347aab7a1596a67b7b00e", + "28063728d31f45aea5beffd3f114eec7", + "d86e65b9ea5f47c58b3f2274e4989f93", + "1b703ddf8c7644f3969c999097242473", + "bcbb16a089ed4e78b2b2e0e6eacabc31", + "ce2ae7abaa4841478a732238269aa233", + "5fd55097dac149579c533b2798ebb442" + ] + }, + "id": "Mkq4RvEq7FQr", + "outputId": "3e99d12a-441e-4914-f48d-8184f8a5b9bc" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "55164fb3b49848beabfe0da534ab4b97", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "README.md: 0%| | 0.00/982 [00:00`!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 175 + }, + "id": "gGFzmplrEy9I", + "outputId": "7acc565b-0759-4438-cd0c-fa68d0570863" + }, + "outputs": [ + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'user\\nWhat is the modulus operator in programming and how can I use it to calculate the modulus of two given numbers?\\nmodel\\nIn programming, the modulus operator is represented by the \\'%\\' symbol. It calculates the remainder when one number is divided by another. To calculate the modulus of two given numbers, you can use the modulus operator in the following way:\\n\\n```python\\n# Calculate the modulus\\nModulus = a % b\\n\\nprint(\"Modulus of the given numbers is: \", Modulus)\\n```\\n\\nIn this code snippet, the variables \\'a\\' and \\'b\\' represent the two given numbers for which you want to calculate the modulus. By using the modulus operator \\'%\\', we calculate the remainder when \\'a\\' is divided by \\'b\\'. The result is then stored in the variable \\'Modulus\\'. Finally, the modulus value is printed using the \\'print\\' statement.\\n\\nFor example, if \\'a\\' is 10 and \\'b\\' is 4, the modulus calculation would be 10 % 4, which equals 2. Therefore, the output of the above code would be:\\n\\n```\\nModulus of the given numbers is: 2\\n```\\n\\nThis means that the modulus of 10 and 4 is 2.\\n'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset[100][\"text\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "idAEIeSQ3xdS" + }, + "source": [ + "\n", + "### Train the model\n", + "Now let's use Huggingface TRL's `SFTTrainer`! More docs here: [TRL SFT docs](https://huggingface.co/docs/trl/sft_trainer). We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 112, + "referenced_widgets": [ + "730aa679b5ac483b929a3646bb5947fa", + "1f333859babd4c1abac69cacda6df864", + "e7f8d2c781a64e83988b0bdd090bdb97", + "3e3feb4fcca74c87abb608c0543236b1", + "2970cbc657d244bab22715bcb788be6a", + "c5b1d1476ddc45249e037df07a96ae37", + "19c27988e01d47e79319f89b5cfd73e2", + "5e48531593d741eaa3669f9118ba8afc", + "62838ffab83d486f86e86417caf0b498", + "7e5378838c114195ba3919fdd683fd7d", + "3350d22f463643ef9a726227f262ac49" + ] + }, + "id": "95_Nn-89DhsL", + "outputId": "2ffe3c70-8c46-41a1-ef51-07c220b5935d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Switching to float32 training since model cannot work with float16\n", + "Unsloth: We found double BOS tokens - we shall remove one automatically.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "730aa679b5ac483b929a3646bb5947fa", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Unsloth: Tokenizing [\"text\"] (num_proc=2): 0%| | 0/100000 [00:00user\\n\",\n", + " response_part = \"model\\n\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Dv1NBUozV78l" + }, + "source": [ + "Let's verify masking the instruction part is done! Let's print the 100th row again:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 175 + }, + "id": "LtsMVtlkUhja", + "outputId": "aebb55c2-3883-4494-e9f8-78d60b9b08e8" + }, + "outputs": [ + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'user\\nWhat is the modulus operator in programming and how can I use it to calculate the modulus of two given numbers?\\nmodel\\nIn programming, the modulus operator is represented by the \\'%\\' symbol. It calculates the remainder when one number is divided by another. To calculate the modulus of two given numbers, you can use the modulus operator in the following way:\\n\\n```python\\n# Calculate the modulus\\nModulus = a % b\\n\\nprint(\"Modulus of the given numbers is: \", Modulus)\\n```\\n\\nIn this code snippet, the variables \\'a\\' and \\'b\\' represent the two given numbers for which you want to calculate the modulus. By using the modulus operator \\'%\\', we calculate the remainder when \\'a\\' is divided by \\'b\\'. The result is then stored in the variable \\'Modulus\\'. Finally, the modulus value is printed using the \\'print\\' statement.\\n\\nFor example, if \\'a\\' is 10 and \\'b\\' is 4, the modulus calculation would be 10 % 4, which equals 2. Therefore, the output of the above code would be:\\n\\n```\\nModulus of the given numbers is: 2\\n```\\n\\nThis means that the modulus of 10 and 4 is 2.\\n'" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.decode(trainer.train_dataset[100][\"input_ids\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4Kyjy__m9KY3" + }, + "source": [ + "Now let's print the masked out example - you should see only the answer is present:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 175 + }, + "id": "_rD6fl8EUxnG", + "outputId": "e9012c2a-60eb-437e-f145-3e11e9a0dd34" + }, + "outputs": [ + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "' In programming, the modulus operator is represented by the \\'%\\' symbol. It calculates the remainder when one number is divided by another. To calculate the modulus of two given numbers, you can use the modulus operator in the following way:\\n\\n```python\\n# Calculate the modulus\\nModulus = a % b\\n\\nprint(\"Modulus of the given numbers is: \", Modulus)\\n```\\n\\nIn this code snippet, the variables \\'a\\' and \\'b\\' represent the two given numbers for which you want to calculate the modulus. By using the modulus operator \\'%\\', we calculate the remainder when \\'a\\' is divided by \\'b\\'. The result is then stored in the variable \\'Modulus\\'. Finally, the modulus value is printed using the \\'print\\' statement.\\n\\nFor example, if \\'a\\' is 10 and \\'b\\' is 4, the modulus calculation would be 10 % 4, which equals 2. Therefore, the output of the above code would be:\\n\\n```\\nModulus of the given numbers is: 2\\n```\\n\\nThis means that the modulus of 10 and 4 is 2.\\n'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.decode([tokenizer.pad_token_id if x == -100 else x for x in trainer.train_dataset[100][\"labels\"]]).replace(tokenizer.pad_token, \" \")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2ejIt2xSNKKp", + "outputId": "ba6de9bc-35f1-48ed-8552-5cf1943d0478" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GPU = Tesla T4. Max memory = 14.741 GB.\n", + "4.283 GB of memory reserved.\n" + ] + } + ], + "source": [ + "# @title Show current memory stats\n", + "gpu_stats = torch.cuda.get_device_properties(0)\n", + "start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\n", + "print(f\"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.\")\n", + "print(f\"{start_gpu_memory} GB of memory reserved.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CNP1Uidk9mrz" + }, + "source": [ + "Let's train the model! To resume a training run, set `trainer.train(resume_from_checkpoint = True)`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "yqxqAZ7KJ4oL", + "outputId": "b44425bc-2ccf-4683-ce72-a837e5a07e9e" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n", + " \\\\ /| Num examples = 100,000 | Num Epochs = 1 | Total steps = 30\n", + "O^O/ \\_/ \\ Batch size per device = 2 | Gradient accumulation steps = 4\n", + "\\ / Data Parallel GPUs = 1 | Total batch size (2 x 4 x 1) = 8\n", + " \"-____-\" Trainable parameters = 14,901,248/4,000,000,000 (0.37% trained)\n", + "It is strongly recommended to train Gemma3 models with the `eager` attention implementation instead of `sdpa`. Use `eager` with `AutoModelForCausalLM.from_pretrained('', attn_implementation='eager')`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Will smartly offload gradients to save VRAM!\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [30/30 16:18, Epoch 0/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
11.237700
21.636400
31.766300
41.420700
51.235700
61.806600
71.010100
81.896600
91.464700
101.309700
111.461600
121.867400
131.854700
141.394800
151.633000
161.238600
172.174100
181.488000
191.521400
201.816000
211.694700
221.614100
231.829700
241.616300
251.269200
261.291600
271.743000
281.525200
291.999000
301.858200

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "trainer_stats = trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pCqnaKmlO1U9", + "outputId": "5d5d33ee-7a84-4418-b038-bd15fb4614e4" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1068.4322 seconds used for training.\n", + "17.81 minutes used for training.\n", + "Peak reserved memory = 13.561 GB.\n", + "Peak reserved memory for training = 9.278 GB.\n", + "Peak reserved memory % of max memory = 91.995 %.\n", + "Peak reserved memory for training % of max memory = 62.94 %.\n" + ] + } + ], + "source": [ + "# @title Show final memory and time stats\n", + "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", + "used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n", + "used_percentage = round(used_memory / max_memory * 100, 3)\n", + "lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n", + "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n", + "print(\n", + " f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\"\n", + ")\n", + "print(f\"Peak reserved memory = {used_memory} GB.\")\n", + "print(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\n", + "print(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\n", + "print(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ekOmTR1hSNcr" + }, + "source": [ + "\n", + "### Inference\n", + "Let's run the model via Unsloth native inference! According to the `Gemma-3` team, the recommended settings for inference are `temperature = 1.0, top_p = 0.95, top_k = 64`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "kR3gIAX-SM2q", + "outputId": "407daa07-ae31-4771-8c31-779665e53bd8" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['user\\nContinue the sequence: 1, 1, 2, 3, 5, 8,\\nmodel\\n13, 21, 34, 55, 89...\\n\\nThis is the Fibonacci sequence, where each number is the sum of the two preceding ones.\\n']" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from unsloth.chat_templates import get_chat_template\n", + "tokenizer = get_chat_template(\n", + " tokenizer,\n", + " chat_template = \"gemma-3\",\n", + ")\n", + "messages = [{\n", + " \"role\": \"user\",\n", + " \"content\": [{\n", + " \"type\" : \"text\",\n", + " \"text\" : \"Continue the sequence: 1, 1, 2, 3, 5, 8,\",\n", + " }]\n", + "}]\n", + "text = tokenizer.apply_chat_template(\n", + " messages,\n", + " add_generation_prompt = True, # Must add for generation\n", + ")\n", + "outputs = model.generate(\n", + " **tokenizer([text], return_tensors = \"pt\").to(\"cuda\"),\n", + " max_new_tokens = 64, # Increase for longer outputs!\n", + " # Recommended Gemma-3 settings!\n", + " temperature = 1.0, top_p = 0.95, top_k = 64,\n", + ")\n", + "tokenizer.batch_decode(outputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CrSvZObor0lY" + }, + "source": [ + " You can also use a `TextStreamer` for continuous inference - so you can see the generation token by token, instead of waiting the whole time!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "e2pEuRb1r2Vg", + "outputId": "de757d2d-a66b-4be6-c9c9-78cf491dfeba" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Okay, let's break down why the sky is blue! It's a fascinating phenomenon that boils down to a combination of physics and light. Here's the explanation:\n", + "\n", + "**1. Sunlight and its Colors:**\n", + "\n", + "* Sunlight, which appears white to us, is actually made up of *all* the\n" + ] + } + ], + "source": [ + "messages = [{\n", + " \"role\": \"user\",\n", + " \"content\": [{\"type\" : \"text\", \"text\" : \"Why is the sky blue?\",}]\n", + "}]\n", + "text = tokenizer.apply_chat_template(\n", + " messages,\n", + " add_generation_prompt = True, # Must add for generation\n", + ")\n", + "\n", + "from transformers import TextStreamer\n", + "_ = model.generate(\n", + " **tokenizer([text], return_tensors = \"pt\").to(\"cuda\"),\n", + " max_new_tokens = 64, # Increase for longer outputs!\n", + " # Recommended Gemma-3 settings!\n", + " temperature = 1.0, top_p = 0.95, top_k = 64,\n", + " streamer = TextStreamer(tokenizer, skip_prompt = True),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uMuVrWbjAzhc" + }, + "source": [ + "\n", + "### Saving, loading finetuned models\n", + "To save the final model as LoRA adapters, either use Huggingface's `push_to_hub` for an online save or `save_pretrained` for a local save.\n", + "\n", + "**[NOTE]** This ONLY saves the LoRA adapters, and not the full model. To save to 16bit or GGUF, scroll down!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "upcOlWe7A1vc", + "outputId": "a99a1086-5a2d-4828-d599-7e3634a069cd" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['gemma-3/processor_config.json']" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.save_pretrained(\"gemma-3\") # Local saving\n", + "tokenizer.save_pretrained(\"gemma-3\")\n", + "# model.push_to_hub(\"HF_ACCOUNT/gemma-3\", token = \"...\") # Online saving\n", + "# tokenizer.push_to_hub(\"HF_ACCOUNT/gemma-3\", token = \"...\") # Online saving" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AEEcJ4qfC7Lp" + }, + "source": [ + "Now if you want to load the LoRA adapters we just saved for inference, set `False` to `True`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MKX_XKs_BNZR", + "outputId": "d016d936-4bd5-40f8-dffa-bcfad987f489" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Okay, let's break down what Gemma-3 is. It's a fascinating development in the world of AI, and here's a comprehensive overview:\n", + "\n", + "**1. What it is:**\n", + "\n", + "* **A Family of Open-Weight Language Models:** Gemma-3 isn't just *one* model\n" + ] + } + ], + "source": [ + "if False:\n", + " from unsloth import FastModel\n", + " model, tokenizer = FastModel.from_pretrained(\n", + " model_name = \"lora_model\", # YOUR MODEL YOU USED FOR TRAINING\n", + " max_seq_length = 2048,\n", + " load_in_4bit = True,\n", + " )\n", + "\n", + "messages = [{\n", + " \"role\": \"user\",\n", + " \"content\": [{\"type\" : \"text\", \"text\" : \"What is Gemma-3?\",}]\n", + "}]\n", + "text = tokenizer.apply_chat_template(\n", + " messages,\n", + " add_generation_prompt = True, # Must add for generation\n", + ")\n", + "\n", + "from transformers import TextStreamer\n", + "_ = model.generate(\n", + " **tokenizer([text], return_tensors = \"pt\").to(\"cuda\"),\n", + " max_new_tokens = 64, # Increase for longer outputs!\n", + " # Recommended Gemma-3 settings!\n", + " temperature = 1.0, top_p = 0.95, top_k = 64,\n", + " streamer = TextStreamer(tokenizer, skip_prompt = True),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f422JgM9sdVT" + }, + "source": [ + "### Saving to float16 for VLLM\n", + "\n", + "We also support saving to `float16` directly for deployment! We save it in the folder `gemma-3-finetune`. Set `if False` to `if True` to let it run!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "iHjt_SMYsd3P" + }, + "outputs": [], + "source": [ + "if False: # Change to True to save finetune!\n", + " model.save_pretrained_merged(\"gemma-3-finetune\", tokenizer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "z6O48DbNIAr0" + }, + "source": [ + "If you want to upload / push to your Hugging Face account, set `if False` to `if True` and add your Hugging Face token and upload location!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZV-CiKPrIFG0" + }, + "outputs": [], + "source": [ + "if False: # Change to True to upload finetune\n", + " model.push_to_hub_merged(\n", + " \"HF_ACCOUNT/gemma-3-finetune\", tokenizer,\n", + " token = \"hf_...\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TCv4vXHd61i7" + }, + "source": [ + "### GGUF / llama.cpp Conversion\n", + "To save to `GGUF` / `llama.cpp`, we support it natively now for all models! For now, you can convert easily to `Q8_0, F16 or BF16` precision. `Q4_K_M` for 4bit will come later!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FqfebeAdT073" + }, + "outputs": [], + "source": [ + "if False: # Change to True to save to GGUF\n", + " model.save_pretrained_gguf(\n", + " \"gemma-3-finetune\",\n", + " quantization_type = \"Q8_0\", # For now only Q8_0, BF16, F16 supported\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Q974YEVPI7JS" + }, + "source": [ + "Likewise, if you want to instead push to GGUF to your Hugging Face account, set `if False` to `if True` and add your Hugging Face token and upload location!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZgcJIhJ0I_es" + }, + "outputs": [], + "source": [ + "if False: # Change to True to upload GGUF\n", + " model.push_to_hub_gguf(\n", + " \"gemma-3-finetune\",\n", + " quantization_type = \"Q8_0\", # Only Q8_0, BF16, F16 supported\n", + " repo_id = \"HF_ACCOUNT/gemma-finetune-gguf\",\n", + " token = \"hf_...\",\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CWSZ7ZSvjeWk" + }, + "source": [ + "Now, use the `gemma-3-finetune.gguf` file or `gemma-3-finetune-Q4_K_M.gguf` file in llama.cpp or a UI based system like Jan or Open WebUI. You can install Jan [here](https://github.com/janhq/jan) and Open WebUI [here](https://github.com/open-webui/open-webui)\n", + "\n", + "And we're done! If you have any questions on Unsloth, we have a [Discord](https://discord.gg/unsloth) channel! If you find any bugs or want to keep updated with the latest LLM stuff, or need help, join projects etc, feel free to join our Discord!\n", + "\n", + "Some other links:\n", + "1. Train your own reasoning model - Llama GRPO notebook [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-GRPO.ipynb)\n", + "2. Saving finetunes to Ollama. [Free notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-Ollama.ipynb)\n", + "3. Llama 3.2 Vision finetuning - Radiography use case. [Free Colab](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(11B)-Vision.ipynb)\n", + "6. See notebooks for DPO, ORPO, Continued pretraining, conversational finetuning and more on our [documentation](https://docs.unsloth.ai/get-started/unsloth-notebooks)!\n", + "\n", + "

\n", + " \n", + " \n", + " \n", + "\n", + " Join Discord if you need help + ⭐️ Star us on Github ⭐️\n", + "
\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "006e1f7432f84b298f61bb597b8aab10": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "018b82e39081491fb1137b8fc2707f03": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1b703ddf8c7644f3969c999097242473", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_bcbb16a089ed4e78b2b2e0e6eacabc31", + "value": 100000 + } + }, + "039f461e15214bd697501219bd9cbbd9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "063db74d47814f95b560bd3bab11b55f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3530e2b431c041c6aeeaca4808ba0424", + "placeholder": "​", + "style": "IPY_MODEL_cc14e51320f34274ae12aa28b04183b7", + "value": " 35.0/35.0 [00:00<00:00, 2.05kB/s]" + } + }, + "083eaaf245954cc998e8f20e8120eed1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0881c055108340f7ab4b840ac1545cbb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "090b146b8c6f4fa1ab0261d2a61be9de": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_708d05af00a64d19bb11dc839a5e68db", + "placeholder": "​", + "style": "IPY_MODEL_3d20e28d74e549c0a43686b214eebd87", + "value": "preprocessor_config.json: 100%" + } + }, + "0a67ecf349484aa18e1151aa2bc1a9ae": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4b1e445cfec04a0f8566be97d8388310", + "placeholder": "​", + "style": "IPY_MODEL_68f1f454dee4462ebf03789f2df9e0f9", + "value": "Unsloth: Standardizing formats (num_proc=2): 100%" + } + }, + "0abe1a2b12e54bb5a07db1a8f3a77738": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0b1768a5b5be4a4d9e14ed9e168044b4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0c3e3fbf02d84114906e939fcca108b5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c4a69698321d435c94211a9dee913c45", + "placeholder": "​", + "style": "IPY_MODEL_36c8135711194884be0e03cb5d3ae7e5", + "value": " 1.61k/1.61k [00:00<00:00, 181kB/s]" + } + }, + "0cba80b626574c11a44c6ce09b5d6e80": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_12d3049cca4a46c08cf5cdfcd5225248", + "placeholder": "​", + "style": "IPY_MODEL_6a9baf0a739c4790baf99b0ccadd6873", + "value": " 33.4M/33.4M [00:00<00:00, 70.2MB/s]" + } + }, + "0db66084c58047aa84edf3b4fbede0c0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_0a67ecf349484aa18e1151aa2bc1a9ae", + "IPY_MODEL_d2ecda1542b24e4fb859cedc23a8e5ef", + "IPY_MODEL_ea31c3d67a98483ca537bc5917b1aee1" + ], + "layout": "IPY_MODEL_083eaaf245954cc998e8f20e8120eed1" + } + }, + "12d3049cca4a46c08cf5cdfcd5225248": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "148125f955954041a8f5631f9338c43e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_54d2fb7e7c2b4107ba758a7c7ef4f382", + "placeholder": "​", + "style": "IPY_MODEL_34da5010faa749e0940c2821f2f46e59", + "value": " 100000/100000 [00:43<00:00, 2627.58 examples/s]" + } + }, + "173033e49f9f46d9930c6afddc983509": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_441ff92725ef40ce807daa1ff721faef", + "placeholder": "​", + "style": "IPY_MODEL_21136eec72034fcbb47a19f92aa0664d", + "value": " 982/982 [00:00<00:00, 78.7kB/s]" + } + }, + "17eaf723882e4efea38119978166fc75": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cebd4fbf1fcf4ab2b65ecf539eda5a1e", + "placeholder": "​", + "style": "IPY_MODEL_f14ea72beac74152af5f970e634769ca", + "value": "generation_config.json: 100%" + } + }, + "18fce8679d7d4961b88bb2162e7aa9eb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f9439c3c9b3b4c4a84ed67aa0601a530", + "placeholder": "​", + "style": "IPY_MODEL_a89575b4c58348ac98566c22ab7e4118", + "value": "chat_template.json: 100%" + } + }, + "19c27988e01d47e79319f89b5cfd73e2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "19c3ef35452d406cb18b72e38b631ee7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_77e843913b6e439ead9cf42725eedf3d", + "max": 35, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4f7e8b6b71484cce80ae9cdf0c481825", + "value": 35 + } + }, + "1b703ddf8c7644f3969c999097242473": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1be74564b60c48d6b21615adf29009fb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ded71beadafd438ebff07bb0594771e4", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_677e4d7a08ab408e9430d67a2870f707", + "value": 100000 + } + }, + "1cb8f93e225f427f92c7bf7aa99aba8c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1ecb60e1d5934ea19a0d2c29aa01158d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1f333859babd4c1abac69cacda6df864": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c5b1d1476ddc45249e037df07a96ae37", + "placeholder": "​", + "style": "IPY_MODEL_19c27988e01d47e79319f89b5cfd73e2", + "value": "Unsloth: Tokenizing ["text"] (num_proc=2): 100%" + } + }, + "21136eec72034fcbb47a19f92aa0664d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "26b61507603d453c8c24af24d301bdb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9e20d704e64a4aaabe5e495c468d9670", + "placeholder": "​", + "style": "IPY_MODEL_ae4e4537ed1d4df4b1677700d190a2d2", + "value": " 192/192 [00:00<00:00, 20.1kB/s]" + } + }, + "28063728d31f45aea5beffd3f114eec7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "28de62814a6847e0a0b41ec6bf8fdc66": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2970cbc657d244bab22715bcb788be6a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2b95af79a12749549da9bd21f0a944db": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2ef58068693547b8b3111d944703d188": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ec4009400bb34f05bd1554db5bbd9ea8", + "IPY_MODEL_018b82e39081491fb1137b8fc2707f03", + "IPY_MODEL_e0387cc0b0d7426d9e6a7a1cd42bb760" + ], + "layout": "IPY_MODEL_3526caa6bc6347aab7a1596a67b7b00e" + } + }, + "2fa2784330b2439ab884ce0966037961": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30918e50f2174d1c8e7af3eef332b6ee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f4b34bc9a62f405383c3b81dc87792b9", + "IPY_MODEL_1be74564b60c48d6b21615adf29009fb", + "IPY_MODEL_148125f955954041a8f5631f9338c43e" + ], + "layout": "IPY_MODEL_91693f16da7b421885fe8474cf533327" + } + }, + "3350d22f463643ef9a726227f262ac49": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "33815b4c0485402e838127b32ad14a15": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_81c3ee8d97d543ae83db3d2748ecd7cd", + "IPY_MODEL_70f9bccb201a4ea79c1827057cd746f6", + "IPY_MODEL_b32538030b8649e7851bfd58bef2786c" + ], + "layout": "IPY_MODEL_0abe1a2b12e54bb5a07db1a8f3a77738" + } + }, + "3477f57107754fb58bf5080869c81092": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9511c493311b4d4e94d5dc0aca4eeffb", + "placeholder": "​", + "style": "IPY_MODEL_f4265abad437426b8efa9091110c77c9", + "value": "train-00000-of-00001.parquet: 100%" + } + }, + "34da5010faa749e0940c2821f2f46e59": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3526caa6bc6347aab7a1596a67b7b00e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3530e2b431c041c6aeeaca4808ba0424": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "353fa47fb98c4070a150edec64503eaa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "360142dac8c54a5eb902078ec42abb65": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_70fdd31291b04dd68d66cf31c03d23ff", + "placeholder": "​", + "style": "IPY_MODEL_6feaf338d39440e78221649ac84af4a6", + "value": "special_tokens_map.json: 100%" + } + }, + "36c8135711194884be0e03cb5d3ae7e5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3a24c19f9a6a4e4dba162062a0562db2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3bbc3e42eb0b4a70bb5bf643c4eb0d40": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c0e4c836aa704fd98349a577c2e6ca15", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_59930ab85ef24940963b2e2c7e578adc", + "value": 100000 + } + }, + "3bfc7d7dd81f49a59fc8ad0d6fff858b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_84c68f2059f247628b672b1079130e9b", + "max": 70, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a88fccae11664baa82418c88639f3521", + "value": 70 + } + }, + "3d20e28d74e549c0a43686b214eebd87": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "3e3feb4fcca74c87abb608c0543236b1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7e5378838c114195ba3919fdd683fd7d", + "placeholder": "​", + "style": "IPY_MODEL_3350d22f463643ef9a726227f262ac49", + "value": " 100000/100000 [03:03<00:00, 568.03 examples/s]" + } + }, + "409488926d2242c5a8e7b3d5b79c59db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_039f461e15214bd697501219bd9cbbd9", + "max": 192, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_9f9d2f43fb5e47df883feb3126fe52e9", + "value": 192 + } + }, + "40bf6f1ffbc5479082fdd7ab153ea974": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "441ff92725ef40ce807daa1ff721faef": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4466f20e614a4cdabe1704859c2f1034": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_be2b1fb954444be089045a378673a958", + "max": 4689074, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_9520485cc7c24c8584c3838717655012", + "value": 4689074 + } + }, + "45616dc4ade54cb28ae49c9489d810ca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "457c60d6a15d4314ba25d370be956a60": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "490baae8c2294c2194f32dae183ce833": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b1e445cfec04a0f8566be97d8388310": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c5e29de7224428bb87de46854ea915a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5e9ba3247edc4fafa7687338424ddccb", + "placeholder": "​", + "style": "IPY_MODEL_97d7e3420e24436cb351b1e9679ff8b6", + "value": " 670/670 [00:00<00:00, 42.7kB/s]" + } + }, + "4e7cbb77617f43e6ae81e42eb7086fb7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f98456d6dbe143448c31378bf4f101d0", + "IPY_MODEL_3bbc3e42eb0b4a70bb5bf643c4eb0d40", + "IPY_MODEL_923d9b71e3f742da94b11731381eb411" + ], + "layout": "IPY_MODEL_58fa3ae2cc264f7cb988a2258bf8b4e3" + } + }, + "4f7e8b6b71484cce80ae9cdf0c481825": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "50881898da2f4b35a288ac9befe5024e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_599375adc5f841d1864166e4d1bd617a", + "IPY_MODEL_3bfc7d7dd81f49a59fc8ad0d6fff858b", + "IPY_MODEL_a0c0025c82394e7fbc6d4cc9a9e9f72f" + ], + "layout": "IPY_MODEL_5f0a37b9edc74cbd822e2e71c6c8a956" + } + }, + "51649c42cf2145f1b1e90c3436805350": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "529996f48ccf404584725386d33401f9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "52f091f99f0442f8bd14a50b7c870c1e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "54d2fb7e7c2b4107ba758a7c7ef4f382": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "54d963afc22a46ae93a6ca4bfaa18cf8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_52f091f99f0442f8bd14a50b7c870c1e", + "placeholder": "​", + "style": "IPY_MODEL_c859133fca324effb73ebc3520e746b6", + "value": " 570/570 [00:00<00:00, 61.1kB/s]" + } + }, + "55164fb3b49848beabfe0da534ab4b97": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_79c551f4fd374acdb112c3d8d82c58a0", + "IPY_MODEL_be024d1de576448fb472c3349b2d6d9e", + "IPY_MODEL_173033e49f9f46d9930c6afddc983509" + ], + "layout": "IPY_MODEL_2b95af79a12749549da9bd21f0a944db" + } + }, + "57ed5097e05f4d92a5c492826f989123": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b5a06422fcac41eb97d2de95f14b1806", + "max": 1157008, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_efa41d07d0fa4adda8025fe9490ed850", + "value": 1157008 + } + }, + "58fa3ae2cc264f7cb988a2258bf8b4e3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5975cf24b18e4082bd80e3f177e0ec15": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b5b8482ef7c44e12a83795e7337521c2", + "IPY_MODEL_ea9045a5c4504a5e96e6a7b13767fe4e", + "IPY_MODEL_0cba80b626574c11a44c6ce09b5d6e80" + ], + "layout": "IPY_MODEL_f344c1ab154b4abdb84b2c221b6162a1" + } + }, + "59930ab85ef24940963b2e2c7e578adc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "599375adc5f841d1864166e4d1bd617a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a15e4524521b42108f49dda23ed56023", + "placeholder": "​", + "style": "IPY_MODEL_a4eaae1b30d442208257c1870d549738", + "value": "processor_config.json: 100%" + } + }, + "5c01ab4767104c0c96c42858317f8877": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5e48531593d741eaa3669f9118ba8afc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5e9ba3247edc4fafa7687338424ddccb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f0a37b9edc74cbd822e2e71c6c8a956": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5fd55097dac149579c533b2798ebb442": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "62838ffab83d486f86e86417caf0b498": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "677e4d7a08ab408e9430d67a2870f707": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "68f1f454dee4462ebf03789f2df9e0f9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "697a187a07204fdfb8556cf5c5028c6b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "69a9d565de6a4d54b8f3989fa8b11941": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_090b146b8c6f4fa1ab0261d2a61be9de", + "IPY_MODEL_d2df1617a4094dd295c49a0a72b269c3", + "IPY_MODEL_54d963afc22a46ae93a6ca4bfaa18cf8" + ], + "layout": "IPY_MODEL_0b1768a5b5be4a4d9e14ed9e168044b4" + } + }, + "6a9baf0a739c4790baf99b0ccadd6873": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6feaf338d39440e78221649ac84af4a6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "708d05af00a64d19bb11dc839a5e68db": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "70f9bccb201a4ea79c1827057cd746f6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "danger", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9be36a19ba86494389904b9fffdc2e48", + "max": 4437712931, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_899bf9a4dc594e178d0b95e3cbe08018", + "value": 4437712508 + } + }, + "70fdd31291b04dd68d66cf31c03d23ff": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "730aa679b5ac483b929a3646bb5947fa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1f333859babd4c1abac69cacda6df864", + "IPY_MODEL_e7f8d2c781a64e83988b0bdd090bdb97", + "IPY_MODEL_3e3feb4fcca74c87abb608c0543236b1" + ], + "layout": "IPY_MODEL_2970cbc657d244bab22715bcb788be6a" + } + }, + "73a283e64f324c27b38216e683050b92": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "740d351b7de241a6acabf6c2853585b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "77907c3444174858bbdc548dee8d0d37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "77c5f8b431ba4c08b8f4d9d8f9fafc16": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_73a283e64f324c27b38216e683050b92", + "placeholder": "​", + "style": "IPY_MODEL_81bddfaa180d4875b5cdb5cc4ae45dab", + "value": "added_tokens.json: 100%" + } + }, + "77e843913b6e439ead9cf42725eedf3d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7850eeab711f494399e82bf96f942888": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "79c551f4fd374acdb112c3d8d82c58a0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_006e1f7432f84b298f61bb597b8aab10", + "placeholder": "​", + "style": "IPY_MODEL_f496e27e3d6446c7997c6fc549de69fa", + "value": "README.md: 100%" + } + }, + "7e5378838c114195ba3919fdd683fd7d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7ff7c3b516d84706a47e91c64b50d171": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "81bddfaa180d4875b5cdb5cc4ae45dab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "81c3ee8d97d543ae83db3d2748ecd7cd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e12bca7d49b149b5b1124c28a669db98", + "placeholder": "​", + "style": "IPY_MODEL_fab7a02d350946ae9563a05cfd04e22b", + "value": "model.safetensors: 100%" + } + }, + "84c68f2059f247628b672b1079130e9b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "86c6d49a55b3477bbccc275dcb55fb52": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_77c5f8b431ba4c08b8f4d9d8f9fafc16", + "IPY_MODEL_19c3ef35452d406cb18b72e38b631ee7", + "IPY_MODEL_063db74d47814f95b560bd3bab11b55f" + ], + "layout": "IPY_MODEL_5c01ab4767104c0c96c42858317f8877" + } + }, + "879c6a0498e54e5c87145c7f7d32de7e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8864799f440c440c8ff8c0696e64215d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8949c35d68a043c5b1384774bb07b2ea": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_94d9900bc8934de688e95e15c9d0c9bb", + "placeholder": "​", + "style": "IPY_MODEL_b469cdf580404f498feb062e4dbad10b", + "value": " 4.69M/4.69M [00:00<00:00, 22.1MB/s]" + } + }, + "899bf9a4dc594e178d0b95e3cbe08018": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "89aa2b1f06ab47d7ac0e4cdfd70d7bfe": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "91693f16da7b421885fe8474cf533327": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "91f1a912832140089e20d35a01de6c74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "923d9b71e3f742da94b11731381eb411": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ebecfe57cabc46028198957fb45114cc", + "placeholder": "​", + "style": "IPY_MODEL_ba9001d2d8db4f75abb557fc2306bf5b", + "value": " 100000/100000 [00:17<00:00, 6582.77 examples/s]" + } + }, + "94d9900bc8934de688e95e15c9d0c9bb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9511c493311b4d4e94d5dc0aca4eeffb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9520485cc7c24c8584c3838717655012": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "95371925c6db4abca3d7987775d1e7b0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "danger", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_be166331dbc5446c8a4f45e861183b71", + "max": 116531415, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_45616dc4ade54cb28ae49c9489d810ca", + "value": 116531404 + } + }, + "97d7e3420e24436cb351b1e9679ff8b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "99685b8ae00c40d5b950cbf44b15efa1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "9be36a19ba86494389904b9fffdc2e48": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9e20d704e64a4aaabe5e495c468d9670": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9f9d2f43fb5e47df883feb3126fe52e9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a0c0025c82394e7fbc6d4cc9a9e9f72f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1ecb60e1d5934ea19a0d2c29aa01158d", + "placeholder": "​", + "style": "IPY_MODEL_f920f6cb263c45d59b851b7c6b631cb5", + "value": " 70.0/70.0 [00:00<00:00, 8.21kB/s]" + } + }, + "a0cf0f8dd74a4d5181d465fa7ff52915": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a15e4524521b42108f49dda23ed56023": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a37b4e454c6743a895f159f963366bc8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_18fce8679d7d4961b88bb2162e7aa9eb", + "IPY_MODEL_fd60bb21ba2e474cb4a6020bd302835e", + "IPY_MODEL_0c3e3fbf02d84114906e939fcca108b5" + ], + "layout": "IPY_MODEL_ab71391446d8482c834429a937f7bb96" + } + }, + "a3ce4f38be9a456c81146fba440c8e3f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a431edec70ad43569fe75342678ad9d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7ff7c3b516d84706a47e91c64b50d171", + "placeholder": "​", + "style": "IPY_MODEL_7850eeab711f494399e82bf96f942888", + "value": " 117M/117M [00:00<00:00, 213MB/s]" + } + }, + "a4a64136f6fc48c799abf1701725534e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c9fbf40fa3dd4ba1b363802dc88764da", + "max": 670, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ae7f1fd06ddc4881934690675891855c", + "value": 670 + } + }, + "a4eaae1b30d442208257c1870d549738": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a74d75b5fbe34eb7b1033d21777ecddb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_3477f57107754fb58bf5080869c81092", + "IPY_MODEL_95371925c6db4abca3d7987775d1e7b0", + "IPY_MODEL_a431edec70ad43569fe75342678ad9d7" + ], + "layout": "IPY_MODEL_b2bf96a0eae14ea78c0530c2574a94c7" + } + }, + "a88fccae11664baa82418c88639f3521": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a89575b4c58348ac98566c22ab7e4118": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a92be3fb752148d887e20afe300f9371": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aa36bff36ae5448b892afea071fc1f1d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "ab71391446d8482c834429a937f7bb96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae4e4537ed1d4df4b1677700d190a2d2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ae7f1fd06ddc4881934690675891855c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "afdb7dfdc17548b39daea1f39d54b45c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b2bf96a0eae14ea78c0530c2574a94c7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b32538030b8649e7851bfd58bef2786c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2fa2784330b2439ab884ce0966037961", + "placeholder": "​", + "style": "IPY_MODEL_51649c42cf2145f1b1e90c3436805350", + "value": " 4.44G/4.44G [00:31<00:00, 142MB/s]" + } + }, + "b469cdf580404f498feb062e4dbad10b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b5a06422fcac41eb97d2de95f14b1806": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b5b8482ef7c44e12a83795e7337521c2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0881c055108340f7ab4b840ac1545cbb", + "placeholder": "​", + "style": "IPY_MODEL_77907c3444174858bbdc548dee8d0d37", + "value": "tokenizer.json: 100%" + } + }, + "ba9001d2d8db4f75abb557fc2306bf5b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc6f29c9a1e14ce8be374867b8be86ac": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bcbb16a089ed4e78b2b2e0e6eacabc31": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "be024d1de576448fb472c3349b2d6d9e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1cb8f93e225f427f92c7bf7aa99aba8c", + "max": 982, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_89aa2b1f06ab47d7ac0e4cdfd70d7bfe", + "value": 982 + } + }, + "be166331dbc5446c8a4f45e861183b71": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "be2b1fb954444be089045a378673a958": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c0e4c836aa704fd98349a577c2e6ca15": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c307400fb17e49ea9d835822e6e22633": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_353fa47fb98c4070a150edec64503eaa", + "placeholder": "​", + "style": "IPY_MODEL_40bf6f1ffbc5479082fdd7ab153ea974", + "value": "tokenizer_config.json: 100%" + } + }, + "c43cef665c9542f982986a74dc50ca98": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c4a69698321d435c94211a9dee913c45": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c5b1d1476ddc45249e037df07a96ae37": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c6c51a350a0b420aab957e3570098e18": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c859133fca324effb73ebc3520e746b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c9fbf40fa3dd4ba1b363802dc88764da": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cc14e51320f34274ae12aa28b04183b7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ce2ae7abaa4841478a732238269aa233": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cebd4fbf1fcf4ab2b65ecf539eda5a1e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d2df1617a4094dd295c49a0a72b269c3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dfac71c6372c46bfad46308bdb04480b", + "max": 570, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_df50d8daa49a4955914704edb89baf61", + "value": 570 + } + }, + "d2ecda1542b24e4fb859cedc23a8e5ef": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_91f1a912832140089e20d35a01de6c74", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_99685b8ae00c40d5b950cbf44b15efa1", + "value": 100000 + } + }, + "d86e65b9ea5f47c58b3f2274e4989f93": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "db7e622bbd0f4357b5687a7c09c1f6fd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_457c60d6a15d4314ba25d370be956a60", + "placeholder": "​", + "style": "IPY_MODEL_bc6f29c9a1e14ce8be374867b8be86ac", + "value": " 1.16M/1.16M [00:00<00:00, 10.7MB/s]" + } + }, + "dd6e4f9b4c6d4260a62b920a6812fd07": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_879c6a0498e54e5c87145c7f7d32de7e", + "placeholder": "​", + "style": "IPY_MODEL_740d351b7de241a6acabf6c2853585b6", + "value": "tokenizer.model: 100%" + } + }, + "ded71beadafd438ebff07bb0594771e4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "df50d8daa49a4955914704edb89baf61": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "dfac71c6372c46bfad46308bdb04480b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e003ee5cf1804ce3928b544b3fa7ba77": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_dd6e4f9b4c6d4260a62b920a6812fd07", + "IPY_MODEL_4466f20e614a4cdabe1704859c2f1034", + "IPY_MODEL_8949c35d68a043c5b1384774bb07b2ea" + ], + "layout": "IPY_MODEL_fca09f95775047efa9d481173f1ba261" + } + }, + "e0387cc0b0d7426d9e6a7a1cd42bb760": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ce2ae7abaa4841478a732238269aa233", + "placeholder": "​", + "style": "IPY_MODEL_5fd55097dac149579c533b2798ebb442", + "value": " 100000/100000 [00:02<00:00, 60707.08 examples/s]" + } + }, + "e12bca7d49b149b5b1124c28a669db98": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e7f8d2c781a64e83988b0bdd090bdb97": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5e48531593d741eaa3669f9118ba8afc", + "max": 100000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_62838ffab83d486f86e86417caf0b498", + "value": 100000 + } + }, + "ea31c3d67a98483ca537bc5917b1aee1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_490baae8c2294c2194f32dae183ce833", + "placeholder": "​", + "style": "IPY_MODEL_3a24c19f9a6a4e4dba162062a0562db2", + "value": " 100000/100000 [00:07<00:00, 13546.66 examples/s]" + } + }, + "ea9045a5c4504a5e96e6a7b13767fe4e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a92be3fb752148d887e20afe300f9371", + "max": 33384568, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_aa36bff36ae5448b892afea071fc1f1d", + "value": 33384568 + } + }, + "eae020595f574192ad9d132853bbf6ec": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_17eaf723882e4efea38119978166fc75", + "IPY_MODEL_409488926d2242c5a8e7b3d5b79c59db", + "IPY_MODEL_26b61507603d453c8c24af24d301bdb9" + ], + "layout": "IPY_MODEL_c6c51a350a0b420aab957e3570098e18" + } + }, + "ebecfe57cabc46028198957fb45114cc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ec4009400bb34f05bd1554db5bbd9ea8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_28063728d31f45aea5beffd3f114eec7", + "placeholder": "​", + "style": "IPY_MODEL_d86e65b9ea5f47c58b3f2274e4989f93", + "value": "Generating train split: 100%" + } + }, + "efa41d07d0fa4adda8025fe9490ed850": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "f14ea72beac74152af5f970e634769ca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f344c1ab154b4abdb84b2c221b6162a1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f373f2c24f3b413aaa9fe1ccfb9c1eab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_360142dac8c54a5eb902078ec42abb65", + "IPY_MODEL_a4a64136f6fc48c799abf1701725534e", + "IPY_MODEL_4c5e29de7224428bb87de46854ea915a" + ], + "layout": "IPY_MODEL_a3ce4f38be9a456c81146fba440c8e3f" + } + }, + "f4265abad437426b8efa9091110c77c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f496e27e3d6446c7997c6fc549de69fa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f4b34bc9a62f405383c3b81dc87792b9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_28de62814a6847e0a0b41ec6bf8fdc66", + "placeholder": "​", + "style": "IPY_MODEL_c43cef665c9542f982986a74dc50ca98", + "value": "Map (num_proc=2): 100%" + } + }, + "f89c08592a25432497bb312f58a13c5c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c307400fb17e49ea9d835822e6e22633", + "IPY_MODEL_57ed5097e05f4d92a5c492826f989123", + "IPY_MODEL_db7e622bbd0f4357b5687a7c09c1f6fd" + ], + "layout": "IPY_MODEL_8864799f440c440c8ff8c0696e64215d" + } + }, + "f920f6cb263c45d59b851b7c6b631cb5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f9439c3c9b3b4c4a84ed67aa0601a530": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f98456d6dbe143448c31378bf4f101d0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a0cf0f8dd74a4d5181d465fa7ff52915", + "placeholder": "​", + "style": "IPY_MODEL_529996f48ccf404584725386d33401f9", + "value": "Map: 100%" + } + }, + "fab7a02d350946ae9563a05cfd04e22b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fca09f95775047efa9d481173f1ba261": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fd60bb21ba2e474cb4a6020bd302835e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_697a187a07204fdfb8556cf5c5028c6b", + "max": 1615, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_afdb7dfdc17548b39daea1f39d54b45c", + "value": 1615 + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/end2end/unsloth_llama3/llama3_1.py b/end2end/unsloth_llama3/llama3_1.py new file mode 100644 index 0000000..b4901df --- /dev/null +++ b/end2end/unsloth_llama3/llama3_1.py @@ -0,0 +1,232 @@ +# Original code from https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-Alpaca.ipynb +import argparse +import torch + + +from datasets import load_dataset +import triton.profiler as proton + +SUPPORTS_BFLOAT16 = torch.cuda.get_device_capability()[0] >= 8 + + +def format_dataset(dataset, tokenizer): + + alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. + + ### Instruction: + {} + + ### Input: + {} + + ### Response: + {}""" + + EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN + def formatting_prompts_func(examples): + instructions = examples["instruction"] + inputs = examples["input"] + outputs = examples["output"] + texts = [] + for instruction, input, output in zip(instructions, inputs, outputs): + # Must add EOS_TOKEN, otherwise your generation will go on forever! + text = alpaca_prompt.format(instruction, input, output) + EOS_TOKEN + texts.append(text) + return { "text" : texts, } + pass + + return dataset.map(formatting_prompts_func, batched = True,) + + + +def train_unsloth(): + from unsloth import FastLanguageModel + from trl import SFTTrainer + from transformers import TrainingArguments + + max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally! + dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ + load_in_4bit = False # Use 4bit quantization to reduce memory usage. Can be False. + + model, tokenizer = FastLanguageModel.from_pretrained( + model_name = "unsloth/Meta-Llama-3.1-8B", + max_seq_length = max_seq_length, + dtype = dtype, + load_in_4bit = load_in_4bit, + # token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf + ) + + dataset = load_dataset("yahma/alpaca-cleaned", split = "train") + dataset = format_dataset(dataset, tokenizer) + + + model = FastLanguageModel.get_peft_model( + model, + r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 + target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj",], + lora_alpha = 16, + lora_dropout = 0, # Supports any, but = 0 is optimized + bias = "none", # Supports any, but = "none" is optimized + # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes! + use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context + random_state = 3407, + use_rslora = False, # We support rank stabilized LoRA + loftq_config = None, # And LoftQ + ) + + + trainer = SFTTrainer( + model = model, + tokenizer = tokenizer, + train_dataset = dataset, + dataset_text_field = "text", + max_seq_length = max_seq_length, + dataset_num_proc = 2, + packing = False, # Can make training 5x faster for short sequences. + args = TrainingArguments( + per_device_train_batch_size = 2, + gradient_accumulation_steps = 4, + warmup_steps = 5, + # num_train_epochs = 1, # Set this for 1 full training run. + max_steps = 60, + learning_rate = 2e-4, + fp16 = not SUPPORTS_BFLOAT16, + bf16 = SUPPORTS_BFLOAT16, + logging_steps = 1, + optim = "adamw_8bit", + weight_decay = 0.01, + lr_scheduler_type = "linear", + seed = 3407, + output_dir = "outputs", + report_to = "none", # Use this for WandB etc + ), + ) + + # @title Show current memory stats + gpu_stats = torch.cuda.get_device_properties(0) + start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) + max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) + print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") + print(f"{start_gpu_memory} GB of memory reserved.") + + trainer_stats = trainer.train() + + # @title Show final memory and time stats + used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) + used_memory_for_lora = round(used_memory - start_gpu_memory, 3) + used_percentage = round(used_memory / max_memory * 100, 3) + lora_percentage = round(used_memory_for_lora / max_memory * 100, 3) + print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.") + print( + f"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training." + ) + print(f"Peak reserved memory = {used_memory} GB.") + print(f"Peak reserved memory for training = {used_memory_for_lora} GB.") + print(f"Peak reserved memory % of max memory = {used_percentage} %.") + print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.") + + +def train_native(): + from trl import SFTTrainer + from transformers import TrainingArguments + from transformers import AutoModelForCausalLM, AutoTokenizer + from peft import get_peft_model + from peft import LoraConfig, TaskType + + + + + max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally! + dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+ + load_in_4bit = False # Use 4bit quantization to reduce memory usage. Can be False. + + model, tokenizer = AutoModelForCausalLM.from_pretrained( + pretrained_model_name_or_path = "unsloth/Meta-Llama-3.1-8B", + # max_seq_length = max_seq_length, + # dtype = dtype, + # token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf + ), AutoTokenizer.from_pretrained("unsloth/Meta-Llama-3.1-8B") + + lora_config = LoraConfig( + r=16, + target_modules=["q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj",], + task_type=TaskType.CAUSAL_LM, + lora_alpha=16, + lora_dropout=0, + use_rslora=False, + # loftq_config=None, + bias="none", + ) + + # lora_model = get_peft_model(model, lora_config) + + dataset = load_dataset("yahma/alpaca-cleaned", split = "train") + dataset = format_dataset(dataset, tokenizer) + + + + trainer = SFTTrainer( + model = model, + # tokenizer = tokenizer, + train_dataset = dataset, + # dataset_text_field = "text", + # max_length = max_seq_length, + # dataset_num_proc = 2, + # packing = False, + peft_config=lora_config, + args = TrainingArguments( + per_device_train_batch_size = 2, + gradient_accumulation_steps = 4, + warmup_steps = 5, + # num_train_epochs = 1, # Set this for 1 full training run. + max_steps = 60, + learning_rate = 2e-4, + fp16 = not SUPPORTS_BFLOAT16, + bf16 = SUPPORTS_BFLOAT16, + logging_steps = 1, + optim = "adamw_8bit", + weight_decay = 0.01, + lr_scheduler_type = "linear", + seed = 3407, + output_dir = "outputs", + report_to = "none", # Use this for WandB etc + ), + ) + + # @title Show current memory stats + gpu_stats = torch.cuda.get_device_properties(0) + start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) + max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) + print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.") + print(f"{start_gpu_memory} GB of memory reserved.") + + trainer_stats = trainer.train() + + # @title Show final memory and time stats + used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3) + used_memory_for_lora = round(used_memory - start_gpu_memory, 3) + used_percentage = round(used_memory / max_memory * 100, 3) + lora_percentage = round(used_memory_for_lora / max_memory * 100, 3) + print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.") + print( + f"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training." + ) + print(f"Peak reserved memory = {used_memory} GB.") + print(f"Peak reserved memory for training = {used_memory_for_lora} GB.") + print(f"Peak reserved memory % of max memory = {used_percentage} %.") + print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.") + + +if __name__ == "__main__": + # parse args + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=str, default="unsloth") + args = parser.parse_args() + + # use args.model to determine which model to train + if args.model == "unsloth": + train_unsloth() + elif args.model == "native": + train_native() diff --git a/microbench.sh b/microbench.sh old mode 100644 new mode 100755 index e426a76..395c430 --- a/microbench.sh +++ b/microbench.sh @@ -17,7 +17,9 @@ do then if [ "$kernel" == "triton" ] then - cmd="time proton -k triton microbench.py --workload $workload --profiler $profiler --kernel $kernel" + # combine workload and kernel into a single argument + profile_name="$workload-$kernel" + cmd="time proton -k triton -n $profile_name microbench.py --workload $workload --profiler $profiler --kernel $kernel" else cmd="time proton microbench.py --workload $workload --profiler $profiler --kernel $kernel" fi From e95435b2d4b11007c94333157f9b190a7683a6da Mon Sep 17 00:00:00 2001 From: Jihyeong Lee Date: Fri, 28 Mar 2025 10:21:18 -0400 Subject: [PATCH 02/11] Added unsloth reqs --- end2end/unsloth_llama3/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 end2end/unsloth_llama3/requirements.txt diff --git a/end2end/unsloth_llama3/requirements.txt b/end2end/unsloth_llama3/requirements.txt new file mode 100644 index 0000000..996eb24 --- /dev/null +++ b/end2end/unsloth_llama3/requirements.txt @@ -0,0 +1,2 @@ +unsloth +peft \ No newline at end of file From 0da538ac71fbc638abc94c8b6cacb49a32a43b86 Mon Sep 17 00:00:00 2001 From: Jihyeong Lee Date: Mon, 12 May 2025 02:49:32 +0000 Subject: [PATCH 03/11] update with various benchmarks --- .gitignore | 4 +- .../alignment/accelerate_config.yaml | 26 + end2end/liger_examples/alignment/run_orpo.py | 35 + end2end/liger_examples/huggingface/README.md | 55 ++ .../liger_examples/huggingface/callback.py | 257 +++++++ .../huggingface/config/fsdp_config.json | 5 + .../huggingface/img/gemma_7b_mem.png | Bin 0 -> 12421 bytes .../huggingface/img/gemma_7b_tp.png | Bin 0 -> 12127 bytes .../huggingface/img/llama_mem_alloc.png | Bin 0 -> 15422 bytes .../huggingface/img/llama_tps.png | Bin 0 -> 16117 bytes .../huggingface/img/qwen_mem_alloc.png | Bin 0 -> 14025 bytes .../huggingface/img/qwen_tps.png | Bin 0 -> 15408 bytes .../huggingface/launch_on_modal.py | 69 ++ .../huggingface/requirements.txt | 6 + .../huggingface/run_benchmarks.sh | 52 ++ .../liger_examples/huggingface/run_gemma.sh | 22 + .../liger_examples/huggingface/run_llama.sh | 21 + .../liger_examples/huggingface/run_qwen.sh | 22 + .../huggingface/run_qwen2_vl.sh | 22 + .../liger_examples/huggingface/training.py | 82 ++ .../huggingface/training_multimodal.py | 176 +++++ end2end/liger_examples/lightning/README.md | 21 + .../liger_examples/lightning/requirements.txt | 8 + end2end/liger_examples/lightning/training.py | 288 +++++++ end2end/liger_examples/medusa/README.md | 72 ++ end2end/liger_examples/medusa/callback.py | 386 ++++++++++ .../docs/images/Memory_Stage1_num_head_3.png | Bin 0 -> 13454 bytes .../docs/images/Memory_Stage1_num_head_5.png | Bin 0 -> 13939 bytes .../docs/images/Memory_Stage2_num_head_3.png | Bin 0 -> 13069 bytes .../docs/images/Memory_Stage2_num_head_5.png | Bin 0 -> 13336 bytes .../images/Throughput_Stage1_num_head_3.png | Bin 0 -> 13950 bytes .../images/Throughput_Stage1_num_head_5.png | Bin 0 -> 14364 bytes .../images/Throughput_Stage2_num_head_3.png | Bin 0 -> 14905 bytes .../images/Throughput_Stage2_num_head_5.png | Bin 0 -> 13879 bytes .../liger_examples/medusa/fsdp/acc-fsdp.conf | 24 + end2end/liger_examples/medusa/medusa_util.py | 267 +++++++ .../liger_examples/medusa/requirements.txt | 3 + .../medusa/scripts/llama3_8b_medusa.sh | 56 ++ end2end/liger_examples/medusa/train.py | 383 +++++++++ end2end/unsloth/unsloth_gemma3/nsys_timing.sh | 13 + end2end/unsloth/unsloth_gemma3/proton_test.py | 101 +++ .../unsloth/unsloth_gemma3/proton_timing.sh | 13 + end2end/unsloth/unsloth_gemma3/pt_timing.sh | 13 + .../unsloth_llama3/Gemma3_(4B).ipynb | 0 .../{ => unsloth}/unsloth_llama3/llama3_1.py | 63 +- end2end/unsloth/unsloth_llama3/nsys_timing.sh | 20 + .../unsloth/unsloth_llama3/proton_timing.sh | 19 + end2end/unsloth/unsloth_llama3/pt_timing.sh | 19 + .../unsloth_llama3/requirements.txt | 0 kernel-microbench/01-vector-add.py | 108 +++ kernel-microbench/01-vector-add_results.csv | 6 + kernel-microbench/02-fused-softmax.py | 226 ++++++ .../02-fused-softmax_results.csv | 6 + kernel-microbench/03-matrix-multiplication.py | 400 ++++++++++ .../03-matrix-multiplication_results.csv | 6 + kernel-microbench/04-low-memory-dropout.py | 130 ++++ .../04-low-memory-dropout_results.csv | 6 + kernel-microbench/05-layer-norm.py | 435 +++++++++++ kernel-microbench/05-layer-norm_results.csv | 6 + kernel-microbench/06-fused-attention.py | 724 ++++++++++++++++++ .../06-fused-attention_results.csv | 6 + kernel-microbench/08-grouped-gemm.py | 350 +++++++++ kernel-microbench/08-grouped-gemm_results.csv | 6 + kernel-microbench/README.rst | 11 + kernel-microbench/all_results.csv | 36 + kernel-microbench/benchmarks.sh | 73 ++ 66 files changed, 5107 insertions(+), 51 deletions(-) create mode 100644 end2end/liger_examples/alignment/accelerate_config.yaml create mode 100644 end2end/liger_examples/alignment/run_orpo.py create mode 100644 end2end/liger_examples/huggingface/README.md create mode 100644 end2end/liger_examples/huggingface/callback.py create mode 100644 end2end/liger_examples/huggingface/config/fsdp_config.json create mode 100644 end2end/liger_examples/huggingface/img/gemma_7b_mem.png create mode 100644 end2end/liger_examples/huggingface/img/gemma_7b_tp.png create mode 100644 end2end/liger_examples/huggingface/img/llama_mem_alloc.png create mode 100644 end2end/liger_examples/huggingface/img/llama_tps.png create mode 100644 end2end/liger_examples/huggingface/img/qwen_mem_alloc.png create mode 100644 end2end/liger_examples/huggingface/img/qwen_tps.png create mode 100644 end2end/liger_examples/huggingface/launch_on_modal.py create mode 100644 end2end/liger_examples/huggingface/requirements.txt create mode 100755 end2end/liger_examples/huggingface/run_benchmarks.sh create mode 100644 end2end/liger_examples/huggingface/run_gemma.sh create mode 100644 end2end/liger_examples/huggingface/run_llama.sh create mode 100644 end2end/liger_examples/huggingface/run_qwen.sh create mode 100644 end2end/liger_examples/huggingface/run_qwen2_vl.sh create mode 100644 end2end/liger_examples/huggingface/training.py create mode 100644 end2end/liger_examples/huggingface/training_multimodal.py create mode 100644 end2end/liger_examples/lightning/README.md create mode 100644 end2end/liger_examples/lightning/requirements.txt create mode 100644 end2end/liger_examples/lightning/training.py create mode 100644 end2end/liger_examples/medusa/README.md create mode 100644 end2end/liger_examples/medusa/callback.py create mode 100644 end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_3.png create mode 100644 end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_5.png create mode 100644 end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_3.png create mode 100644 end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_5.png create mode 100644 end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_3.png create mode 100644 end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_5.png create mode 100644 end2end/liger_examples/medusa/docs/images/Throughput_Stage2_num_head_3.png create mode 100644 end2end/liger_examples/medusa/docs/images/Throughput_Stage2_num_head_5.png create mode 100644 end2end/liger_examples/medusa/fsdp/acc-fsdp.conf create mode 100644 end2end/liger_examples/medusa/medusa_util.py create mode 100644 end2end/liger_examples/medusa/requirements.txt create mode 100644 end2end/liger_examples/medusa/scripts/llama3_8b_medusa.sh create mode 100644 end2end/liger_examples/medusa/train.py create mode 100755 end2end/unsloth/unsloth_gemma3/nsys_timing.sh create mode 100644 end2end/unsloth/unsloth_gemma3/proton_test.py create mode 100755 end2end/unsloth/unsloth_gemma3/proton_timing.sh create mode 100755 end2end/unsloth/unsloth_gemma3/pt_timing.sh rename end2end/{ => unsloth}/unsloth_llama3/Gemma3_(4B).ipynb (100%) rename end2end/{ => unsloth}/unsloth_llama3/llama3_1.py (71%) create mode 100755 end2end/unsloth/unsloth_llama3/nsys_timing.sh create mode 100755 end2end/unsloth/unsloth_llama3/proton_timing.sh create mode 100755 end2end/unsloth/unsloth_llama3/pt_timing.sh rename end2end/{ => unsloth}/unsloth_llama3/requirements.txt (100%) create mode 100644 kernel-microbench/01-vector-add.py create mode 100644 kernel-microbench/01-vector-add_results.csv create mode 100644 kernel-microbench/02-fused-softmax.py create mode 100644 kernel-microbench/02-fused-softmax_results.csv create mode 100644 kernel-microbench/03-matrix-multiplication.py create mode 100644 kernel-microbench/03-matrix-multiplication_results.csv create mode 100644 kernel-microbench/04-low-memory-dropout.py create mode 100644 kernel-microbench/04-low-memory-dropout_results.csv create mode 100644 kernel-microbench/05-layer-norm.py create mode 100644 kernel-microbench/05-layer-norm_results.csv create mode 100644 kernel-microbench/06-fused-attention.py create mode 100644 kernel-microbench/06-fused-attention_results.csv create mode 100644 kernel-microbench/08-grouped-gemm.py create mode 100644 kernel-microbench/08-grouped-gemm_results.csv create mode 100644 kernel-microbench/README.rst create mode 100644 kernel-microbench/all_results.csv create mode 100755 kernel-microbench/benchmarks.sh diff --git a/.gitignore b/.gitignore index f7fcfb3..1f00185 100644 --- a/.gitignore +++ b/.gitignore @@ -162,7 +162,7 @@ cython_debug/ #.idea/ -end2end/unsloth_llama3/outputs/* -end2end/unsloth_llama3/unsloth_compiled_cache/* +outputs/ +unsloth_compiled_cache/ *.hatchet *.nsys-rep diff --git a/end2end/liger_examples/alignment/accelerate_config.yaml b/end2end/liger_examples/alignment/accelerate_config.yaml new file mode 100644 index 0000000..e70f3cd --- /dev/null +++ b/end2end/liger_examples/alignment/accelerate_config.yaml @@ -0,0 +1,26 @@ +compute_environment: LOCAL_MACHINE +debug: false +distributed_type: FSDP +downcast_bf16: 'no' +enable_cpu_affinity: false +fsdp_config: + fsdp_activation_checkpointing: false + fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP + fsdp_backward_prefetch: BACKWARD_PRE + fsdp_cpu_ram_efficient_loading: true + fsdp_forward_prefetch: false + fsdp_offload_params: false + fsdp_sharding_strategy: FULL_SHARD + fsdp_state_dict_type: SHARDED_STATE_DICT + fsdp_sync_module_states: true + fsdp_use_orig_params: true +machine_rank: 0 +main_training_function: main +num_machines: 1 +num_processes: 8 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false diff --git a/end2end/liger_examples/alignment/run_orpo.py b/end2end/liger_examples/alignment/run_orpo.py new file mode 100644 index 0000000..2d734f8 --- /dev/null +++ b/end2end/liger_examples/alignment/run_orpo.py @@ -0,0 +1,35 @@ +import torch + +from datasets import load_dataset +from transformers import AutoModelForCausalLM +from transformers import AutoTokenizer +from trl import ORPOConfig # noqa: F401 + +from liger_kernel.transformers.trainer import LigerORPOTrainer # noqa: F401 + +model = AutoModelForCausalLM.from_pretrained( + "meta-llama/Llama-3.2-1B-Instruct", + torch_dtype=torch.bfloat16, +) + +tokenizer = AutoTokenizer.from_pretrained( + "meta-llama/Llama-3.2-1B-Instruct", + max_length=512, + padding="max_length", +) +tokenizer.pad_token = tokenizer.eos_token + +train_dataset = load_dataset("trl-lib/tldr-preference", split="train") + +training_args = ORPOConfig( + output_dir="Llama3.2_1B_Instruct", + beta=0.1, + max_length=128, + per_device_train_batch_size=32, + max_steps=100, + save_strategy="no", +) + +trainer = LigerORPOTrainer(model=model, args=training_args, tokenizer=tokenizer, train_dataset=train_dataset) + +trainer.train() diff --git a/end2end/liger_examples/huggingface/README.md b/end2end/liger_examples/huggingface/README.md new file mode 100644 index 0000000..41de0dc --- /dev/null +++ b/end2end/liger_examples/huggingface/README.md @@ -0,0 +1,55 @@ +# Liger-Kernel Example with HuggingFace Trainer + +## How to Run + +### Locally on a GPU machine +You can run the example locally on a GPU machine. The default hyperparameters and configurations work on single node with 4xA100 80GB GPUs. + +```bash +pip install -r requirements.txt +sh run_{MODEL}.sh +``` + +### Remotely on Modal +If you do not have access to a GPU machine, you can run the example on Modal. Modal is a serverless platform that allows you to run your code on a remote GPU machine. You can sign up for a free account at [Modal](https://www.modal.com/). + +```bash +pip install modal +modal setup # authenticate with Modal +modal run launch_on_modal.py --script "run_qwen2_vl.sh" +``` + +**Notes** +1. This example uses an optional `use_liger` flag. If true, it does a 1 line monkey patch to apply liger kernel. +2. The example uses Llama3 model that requires community license agreement and HuggingFace Hub login. If you want to use Llama3 in this example, please make sure you have done the followings: + * Agree on the community license agreement https://huggingface.co/meta-llama/Meta-Llama-3-8B + * Run `huggingface-cli login` and enter your HuggingFace token +3. The default hyperparameters and configurations work on single node with 4xA100 80GB GPUs. For running on device with less GPU RAM, please consider reducing the per-GPU batch size and/or enable `CPUOffload` in FSDP. + + +## Benchmark Result + +### LLaMA +Benchmark conditions: LLaMA 3-8B, Alpaca Dataset, Max seq len = 512, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 4 A100s. + +Throughput improves by around 20%, while GPU memory usage drops by 40%. This allows you to train the model on smaller GPUs, use larger batch sizes, or handle longer sequence lengths without incurring additional costs. + +![Throughput](img/llama_tps.png) +![GPU Memory Allocated](img/llama_mem_alloc.png) + +### QWEN +Benchmark conditions: Qwen2-7B, Alpaca Dataset, Max seq len = 512, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 4 A100s. + +Throughput improves by around 10%, while GPU memory usage drops by 50%. + +![Throughput](img/qwen_tps.png) +![GPU Memory Allocated](img/qwen_mem_alloc.png) + + +### GEMMA 7B +Benchmark conditions: Gemma-7B, Alpaca Dataset, Max seq len = 512, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 4 A100s. + +Throughput improves by around 24%, while GPU memory usage drops by 33%. + +![Throughput](img/gemma_7b_mem.png) +![GPU Memory Allocated](img/gemma_7b_tp.png) diff --git a/end2end/liger_examples/huggingface/callback.py b/end2end/liger_examples/huggingface/callback.py new file mode 100644 index 0000000..c834fc5 --- /dev/null +++ b/end2end/liger_examples/huggingface/callback.py @@ -0,0 +1,257 @@ +import time + +from dataclasses import dataclass + +import torch +import transformers + +from transformers import TrainerControl +from transformers import TrainerState +from transformers import TrainingArguments + +from liger_kernel.utils import infer_device + +# https://simple.wikipedia.org/wiki/Byte +# For memory, we use binary system +M_BIN_UNIT = 2**20 +# For metrics (tflops), we use decimal system +T_DEC_UNIT = 10**12 + + +def round_to_n_decimal(x, n): + return round(x, n) + + +@dataclass +class Precision: + """ + Precision is a dataclass to store the number of decimal points for each metric. + """ + + n_decimal_time: int + n_decimal_memory: int + n_decimal_TPS: int + + +@dataclass +class State: + """ + State is a dataclass to store the internal state of the efficiency callback. + """ + + n_warmup_steps: int = 0 + total_peak_memory_allocated: float = float("-inf") + total_peak_memory_reserved: float = float("-inf") + + step_start_time: float = 0.0 + elapsed_time: float = 0.0 + + elapsed_step: int = 0 + + step_start_tokens_seen: int = 0 + elapsed_tokens_seen: int = 0 + + global_start_step: int = 0 + + +@dataclass +class Time: + """ + Time is a dataclass to store the time-related metrics. + """ + + step: int = 0 + step_time_sec: float = 0.0 + avg_step_time_sec: float = 0.0 + time_to_completion_sec: float = 0.0 + estimated_total_time_sec: float = 0.0 + + +@dataclass +class Memory: + """ + Memory is a dataclass to store the memory-related metrics. + """ + + step_peak_memory_allocated_MB: float = 0.0 + step_peak_memory_reserved_MB: float = 0.0 + total_peak_memory_allocated_MB: float = 0.0 + total_peak_memory_reserved_MB: float = 0.0 + + +@dataclass +class TPS: + """ + TPS is a dataclass to store the tokens per second metrics. + """ + + step_tokens_per_second: float = 0.0 + avg_tokens_per_second: float = 0.0 + + +class EfficiencyCallback(transformers.TrainerCallback): + """ + EfficiencyCallback is a callback to track the efficiency of the training process. + The tracked stats include: step time, memory, and throughput. + + It requires including `--include_num_input_tokens_seen` and `logging_steps=1` in the training arguments. + + Args: + n_warmup_steps: number of warmup steps + The stats in the first n_warmup_steps will not be added into the aggregated stats + This is because the first few steps might take longer due to jit compliation and other initialization overheads + n_decimal_time: number of decimal points for time + n_decimal_memory: number of decimal points for memory + n_decimal_TPS: number of decimal points for TPS + """ + + def __init__(self, n_warmup_steps=2, n_decimal_time=2, n_decimal_memory=2, n_decimal_TPS=2): + self.state = State( + n_warmup_steps, + ) + + self.precision = Precision(n_decimal_time, n_decimal_memory, n_decimal_TPS) + + self.time = Time() + self.memory = Memory() + self.tps = TPS() + self.device = infer_device() + + def on_init_end( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ): + """ + Event called at the end of the initialization of the [`Trainer`]. + """ + if not args.include_num_input_tokens_seen: + raise Exception( + 'Please pass training argument "--include_num_input_tokens_seen" to track tokens per second' + ) + if args.logging_steps != 1: + raise Exception("Please set logging_steps=1 to track the efficiency metrics accurately") + + def on_train_begin( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ): + # if loaded from checkpoints, global_start_step is not 1 but state.global_step + self.state.global_start_step = state.global_step + + def on_log( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + logs: dict[str, float], + **kwargs, + ): + if state.global_step < (self.state.global_start_step + self.state.n_warmup_steps): + return + else: + # spread self.time, self.memory, self.tps to logs + logs.update(self.time.__dict__) + logs.update(self.memory.__dict__) + logs.update(self.tps.__dict__) + + def on_step_begin( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ): + """ + Event called at the beginning of a training step. If using gradient accumulation, one training step might take + several inputs. + """ + # memory + getattr(torch, self.device).reset_peak_memory_stats() + + # time + self.state.step_start_time = time.perf_counter() + + def on_step_end( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ): + if state.global_step < (self.state.global_start_step + self.state.n_warmup_steps): + # The end the current step_start_tokens_seen is the start of next iteration + + # tokens + self.state.step_start_tokens_seen = state.num_input_tokens_seen + return + + # time + current_time = time.perf_counter() + step_time = current_time - self.state.step_start_time + self.state.elapsed_time += step_time + + # step + global_step = state.global_step + self.state.elapsed_step += 1 + avg_step_time = self.state.elapsed_time / self.state.elapsed_step + + self.time.step = global_step + self.time.step_time_sec = round_to_n_decimal(step_time, self.precision.n_decimal_time) + self.time.avg_step_time_sec = round_to_n_decimal(avg_step_time, self.precision.n_decimal_time) + self.time.time_to_completion_sec = round_to_n_decimal( + avg_step_time * (state.max_steps - global_step), + self.precision.n_decimal_time, + ) + self.time.estimated_total_time_sec = round_to_n_decimal( + avg_step_time * state.max_steps, self.precision.n_decimal_time + ) + + # memory + step_peak_memory_allocated = getattr(torch, self.device).memory.max_memory_allocated() + step_peak_memory_reserved = getattr(torch, self.device).memory.max_memory_reserved() + + self.memory.step_peak_memory_allocated_MB = round_to_n_decimal( + step_peak_memory_allocated / M_BIN_UNIT, self.precision.n_decimal_memory + ) + self.state.total_peak_memory_allocated = max(self.state.total_peak_memory_allocated, step_peak_memory_allocated) + self.memory.total_peak_memory_allocated_MB = round_to_n_decimal( + self.state.total_peak_memory_allocated / M_BIN_UNIT, + self.precision.n_decimal_memory, + ) + + self.memory.step_peak_memory_reserved_MB = round_to_n_decimal( + step_peak_memory_reserved / M_BIN_UNIT, self.precision.n_decimal_memory + ) + + self.state.total_peak_memory_reserved = max(self.state.total_peak_memory_reserved, step_peak_memory_reserved) + + self.memory.total_peak_memory_reserved_MB = round_to_n_decimal( + self.state.total_peak_memory_reserved / M_BIN_UNIT, + self.precision.n_decimal_memory, + ) + + # tokens + step_tokens_seen = state.num_input_tokens_seen - self.state.step_start_tokens_seen + + self.state.elapsed_tokens_seen += step_tokens_seen + + self.tps.step_tokens_per_second = round_to_n_decimal( + step_tokens_seen / step_time, + self.precision.n_decimal_TPS, + ) + + self.tps.avg_tokens_per_second = round_to_n_decimal( + self.state.elapsed_tokens_seen / self.state.elapsed_time, + self.precision.n_decimal_TPS, + ) + + # The end the current step_start_tokens_seen is the start of next iteration + + # tokens + self.state.step_start_tokens_seen = state.num_input_tokens_seen diff --git a/end2end/liger_examples/huggingface/config/fsdp_config.json b/end2end/liger_examples/huggingface/config/fsdp_config.json new file mode 100644 index 0000000..45894b0 --- /dev/null +++ b/end2end/liger_examples/huggingface/config/fsdp_config.json @@ -0,0 +1,5 @@ +{ + "backward_prefetch": "backward_pre", + "forward_prefetch": "true", + "activation_checkpointing": true +} \ No newline at end of file diff --git a/end2end/liger_examples/huggingface/img/gemma_7b_mem.png b/end2end/liger_examples/huggingface/img/gemma_7b_mem.png new file mode 100644 index 0000000000000000000000000000000000000000..940d0918bd2c74aaff45666b5d5b4f4778f3fb56 GIT binary patch literal 12421 zcmeHtc~sNq)^60It%#_gh$5*)1!aiHJVr!CWL7I61W-f@3KSW_lwh$AfI<-jWr~Un zDwD_X!~5=c?`QA* zJUiae-b!ZewzUulM8^8q;S&(ZN*4%ZMaZ|S!8cSIvJwK>Z)<(n;#7nOV>EVTGz=jQ z4s6lfYNKi{ETdWUwP%((}D4s)o%mSw7yZ*Yrenxhck~3o!h-_ zQ{1W&HMMm<+Y59=yAjuDP9}HY=No#eW>D>uABX2qX0tVNwcp54Un4w{~zQxjo ziVg?X(bApZEfKqq;0ir3mtyCgWA*w4IVS}#T+&nF1 z0@t{kMCMj=N`g=q`?*L83OQ$EHKGc6DZ8%9tICZ1T7^!IX4=Jyh_R!^^&cxbkCs5M zqj_To+kynaZn0usEJbKUPR&3&G@1`xN5!tL6E!gECUZ$?*BYaFNoSx$Z+^mN_0`zX z;kG9EBYbuzjQC@YX#v)>iBcUv>Vq$h!Sy@Gj-@Yh3gJ0qUK3A`v=XxZT=QEht=Wah z%#SZ+FwyJIbtLWRw#oLzpu&giLVV|Z-IHm@mv`CH1iw0hChNZq7xEcFvC;jDh9@?4 zihqYLoqa$m8MQMQ?KjcIy#uoiWH2FP4d?&HS!8pxLS2^Dd>F=;`?l$ri#(ES+{yEU%uD z)AZ>Yj3aV|q8EOj@Qo(clkfJJ&UJAf%A$&xb$Yv#t(#zsN|^&!Ye%1zHDT*KoeamG z$NPu^k*1NPte~bkK3EnXV^xwo^IT}-x!6}*;}(UPBX(>|ucc^<4OKr;mKc=Gd6r=Z zenYaS&=w;YnQR|9N*`hKqCWhxrVlvn(gf{Aq87Hubqh~tY5h|}?Dm;4eV*!UY?ppu z4F>oIE4>4z@Z3}xgYJQgg%eG|^3sq!54{3LYM;9*h&TAMe$AV_HXbvs0#%&FYsD4jjBK2t}iFW?CIvOn)O2r!n?QIHLS`_k%8137GE)Ck9 zS|pZvQZ6a9m12aAvK(z0tf_Jb2u=gsz1GW`0j}AG(3v2dC>$XLIoV^O?h+j8-Ev)~ z@zp!?*z=6xs_qJDt=b2By_{n71BUz6^;x8dEzAoEKCBamM=mHu-pt$ZTqEgtWfjxP z)S}&F>sVCwngXl(_7Dn+Hy>z%!EcIfp9gVH;`kD0__UmbMK%9maZo*@n#^cRx%LsD z0zAtn_>@Z0nYN9;x}O_vqaTE(R$E-yz}%YX6a0-@uI-pWgcl1Pm}pMD2t7<{H?m+= z%#CjBO-_~qnn-Gwf!Sadt2Eqlyux=1aZ!KX|b9sykn`+U$ zkWrDSV7s<0g;V9Jn-;mxm$CR z4Xctf#{*QcxHas1MZg1j+Ua!s23|-5iCNefV6Hj^6SHAi60b`}7SgMSzULXi2y$J5 z(LfySYF9u5bMv{=!ER2k*(v?P3eScvqm&qUd-VO%H7G%yOieqNe)L&byr6<1KEoLV9r3{xf@qELir6(BJTL%PW3yO ze%D6o^%kHp2J&p*sJ8GiGe7#g@WphUDSy{AH_{Ti@N!M24?`|<&YH&vDOc|Hz5}p6 z=1?`-_K5s-Rv->Ks;`h7Gy6`0;z{)ZTgT0Lf<=pT*#icmGj}~blFIl@@e*_5vi#U4 zzTFQ_4+OnTvng*Zj<&N0mk=iMSlyMM8&VGWuwkzYKcs+~eLq6=L__E%)vorGhc4u$ zevWW^q>!lRq(r?cKUO0anvr%}GOdK%9Z=bgoqM7R3E=5seNvT!$E+$&Hv`|>$9zJ> zEwsV7=s`5|=*5h5N$L+uZ{weGVpG3u?tm|`w0W_=b?m~m%EMx-!EP2@uDWbFqE~$z zvomSago+yjgNzL&3Ez(kB3fh&ZvFG7hS5|S)^NxSH?r+o?ct@}J3XGxSmAgNlIanh zV>Fy$2Tbg2!%C6ahqF0B*{~9FvUjIbITgxenR~0{wvxm~p)n0t0}1x%j#E=eICb>g zbB)}e+fCMPVDl82Vx=y&mDG50&O5M)G1GijPeww=15}`${B_(yB5vyog#1%(=k=?m zd+NN74qNS%L~BW>ILy%mv)Er4m0>eg&G#wJ<{DXJ|LG)#(U*;Gwc0M`lnpqbRlTyy z$;oJ@1(d*Lvo#-*#!ih{!_Xm3ON%pEOhuTd&)8^kjQj)2GZVAO`dMELGyJ=_7EP1dO|)pS~Ra*|Hv>%-Q02?`S&8sT6obyL;{x| zmb!b^vf!XPPzwhRP)Kze?w635=V-Vm!K2WOz`*A$gVHObxV;&c9 zNnBndx29BD6etGezCAOFSRcxI$ElZHbI7Lu@ZRS*K{YN6F@+SAzF4vTM|q7?Hq!lv z@4wzo{&YQbG>kZ3D(;1foE8T>MFSIfjX@micF;!ze%N78OT;-vrhbPTdjAwJN%d1r z15MM9(sY%#Et7m>Rlbz}qqM2%rXAmk$UXn`;3Eo%wB`Dbt&(hdCkei^z>ySje4Wx8 zF5;Brn8$V5n1fV&c8ut)dYLUgWf$9T4OtWN?YPBW5x&RHbln(7?NqDN=?xC~PC z%(;3V4qTdc(w?q)v8>aa`#d>cf{)7cCnmnH*-jh_Heg>vvd?&~OUDX@$Rb~Wk7>#>!h)hnq)gTBh(otjE;GV9 zGhhysg?58JNl|vL_FP*68T)>=&jw~kst%X|YnFzJrmP5M!nb9*@F0?8&3NOP;>wW8 z9_{3z!!-mcO-H%>)}`6)FIEVh${jVqN1q?+kxYAuEwehAXkI;mA{wRWWsKoN@Mbl2 z!Z?M72SZwwn1dX4gjGQc3`i>vQcwO%U%^>Ny?%$ZXQy z{t_xBJu^pT%9lp*^q}!gKZxV zpP$@1z7%Ewd|Tr3gB~51AP%3MeJ`sA+4Lgv&4gZAWOoq_$J&ldS^6X^-k%fFW7O!w zQ2mXiH8XE`ZZk~wx*=~*PyL%=(vc@5O2CEXLsQNE`4Cg>4QTF+{1HkPicIhC_h3b8 z-|Y1GVX>Pc>~>61^Z6%E zoiNZ8UxKrK(A~BGchK&4^A9{9ji{j% zP@a`EY9*a>|G8PJJJLLu)S(n&8-#GjF{(+K8RlcOcNsV#tuZXW2HG@V5|rvrU^S71EOJ^dx3ciI zYbkN;^{H$H9p9ep2{l)=;(>9wn;6zD-^UIyRJk!S+U*hn7KuowJgN3-jw^WIR^L<+ zL7zjSCy{-4VHf_f0!aOIE-8pvLcrvgP_XKywT~kEWUh|PE|tn&Hd%9Aa!e0(&gvgT zsJRrSZz+n8gU0+26SJq!uP~?JS%I#^legZJ`EU(jB11I%td9&cFVPU!FiwTHEBfkM zwNrPEJ2y}_(?J)V6nG$CQ{2@;*Mp9@roFGW0Wo~||0?ezQ{sh)70uJjTaHqM*sz(t zk-8AVEk&KaSJ$Ko(*w_LVAn$+UI*5dc;&>f-*~Sbq_!mz-riLw43@YNDttQo9vw-% z{`s1GueRQ23XM}oyw{#q@*A%3u?dL;eWc%#kZ*}t1E@8{>nF;oQv)&vBl`=yK>4YD z`_?Vp3?yUt?uDLgqy8|E!crg3#obNzkkWLy0YyhJ*-a=xU!@=6^D9C<%Gtfo9!OmT zyoDb=*Q}dnI4p04xn5@Et)pRNUK47@=NnJzm zSi5czx#3~yDhTA>x*UaKeY3c#P(K`SX1Q%5OPoVeH~Rh zjCo@u&ok!wzO+1SniG@J?4>mu2{aOKITwQAj zzJo&`dJnvQJa2ned)*)Z9_~!=hCqr|NIm=E%xR@nkb^R6E}MHRs9XQI_h$O$4hY0? z)!NfXDvJ$QL#}LdEZ_IYr%f9I>Aw>9<3>&Oq!o|{tL2LmesT@||L&0oy91lNgNzxg zObD>t`WCHV+}Kjyx-X{EGhrOUU%o5HgZg~2S<(G1TRI;6*A)a{$h3!EKQ3GO!3AL9 zs|UW=s^1ov-tqcpBVNh$0|tEi#k!6!k&AaaDn%*idS`YJp0U;Uj*CXmLJousg@Zp!=S-il%$*{t}PIsG^yIZIT22yf)zM zl9D6z&F+lH0i^2uWfI19=r02>@aeJ-72TIC&301iS%84c=uRotNYV`$s>rWvINNev zH+D(*ur$km*|GlEp-yF76>$Af?vQWcNe8p4noEPlu72mwKdx)ING#P2ndtg*!@YF| zl;;LH++T9MbB`%);+f3CO{avwyq0@iWX%`6j2EZ!2pl?r@S`(gzztA)H+Wi1K>ylh z=?8q-*Pa$Xg{bsr*|p+N4_<_SQWu<_GJckzd zLj53e_FyT(EdZ?eXzZRt>yL82Yt>%VNwL&1Ht3kbpE?QR+^0xNy{Fk2u(i=d1r3+q zBovi9xi;84(;$fKNVLrc)ze!65Pt=|Vt^*-FZb3{P}q)EgU*KDV0B zSTG?hrgdf%?zTx^7dCn-4>-ch^Y2tQbiBOU)2*#Wh-}94C$b2^9Xm3TjHio2-|j7Z zY{G1KQ4cb%(KD^;iXdfgv~;&ot3?xyz#i^++tY4irKvusJ^?pKs-<XV-;(4hSn1f2IWWjF#^tXT9%4gU9`(cuk`E0ZEfBI?or5mlHB!Mi{*z)t? zeYvC%G38o%nYz?+qX+-bhj4%PUD7YIIXR!t>ob%b7=5^771Y?|tH)q1O;($Sh9~{9 zCMWApWO);MBYa)V)W?55wp99h#@2;*Lq@BWLj(T=nmvRa!`h!7f>`*&HUBtI+X9nH z?+=!~lUpO+)fs)koqM*(pE_RN=;`@;!T%u(1N;rd(hEX`td2s^{~gGQpvLvWG?=G14^!M2%A zZ6p+q@x(pv(BLyQfrv~KWMHPY_t7(et~}EiHUGJ8{8tzlWYkb{bLWeF^IO($k0$*0 zFIX?2b(a^uczaNN`)PIb z78ow-c7^%RO?3MK<##}m(%AS%pdbk~AAffnR3BfL-#F5efW;2Fb9uTJBJ*^yOG^01_bKTx&)jqk*8~670?W_8(4DCZC>&kX zB4=VWmZf;Q#r@yx===A571Xca`?<#cyS)3{D}UwLpBvw=ljYx7|2NkEk79j$lNT7r z(E5rfIb(4D0H6fE_ExUcMTy8ldwW+RwD=8G`1ticI)Z(Z_7L>&Uq$|Nm`gCnAC%qQ zG@0%CkNIQWS_&@uTmtK57zB>o0|Q&CZh#wAatB+a z_vQPwC2aNsQ(=C5HqGzNo!!{)Adr_9KAzpjA_M0n!SwXJ@Lkf~% zK}(sxnjC!fINo4aHfU_ub9(D zDVv0g`ShZ^q~oxd-=zsVkR#guX8Ugfy`#VY$M5c@6h_Dm0*3cN7Dp6)6T2T_o2llQ z;eOeAxH`ZM`yoy`^M|`;BD#ZvvnMJ{qBZqJPUd9T8`B>TI5p{Kh1g?&bqI+boJUSGzgW0O{CJ}*JKdIY^25O+XFcn~OSEPcnarKjuR^uy)bAJ{U*PQv zbEBS?`nRfb;FCu9-nN%7!QBn+c@CFg5zW$ldPH}E5a~wrh}N{SWfhq&v63Amy;9W3 zm@-jFn*U?|jC3KgD9peEMlTZ?y&HeP9!R zLf5UmABFNWAOA=KlWPBgyYTHH)(`~FfbAX%A>ICdoVA6RXfnfO* zlr~Wf%*>d(J+_&OIE1W>fGbw&xm~(L#jY>;!pN$_Puv9t;TfS z?IqM(%vC+WJWe>6BW040%FjzAR5}8fiiu)IW34U4%Vz$c7=-=nDcn=uK#ahC z3)v$jCH-kzs}M*`zbW@}X%!90mQ(4=f7 zL!h}pzczRb=J9xg8RYd>p2WxQgeaXDZu6P!$)PvQ^{DT&fB2Y_c1cbmau!HN+U%Z4 zM0EmFL}un2G}7&`iV)6^T$l_*Y1f%6vpDe*=Y5BHHB!{+xpX;-JeRvnRdP<@p%8?qN2E^>g0#^}NqhR{ zfJE9=adllXXE3+l*AF=JA_i!?bRwET>>E?&)p8P?8-6fb_;9tcOv({xb5|^n*Lxj# z5($)6!mh+TR}8<)w+|Gh*;-8tR9#SRlCfOk(YkmQN6&><3Vj41*Ex}d%}}95Na)7| z_0pe?pN2ankcATlg*IAsyGlH99?r>o8|*w z(;=t!C5$>ZvG0gWk$>_bQQ0GI`!MF~x(ay4E#Yn)?DS zcmYJ(vcwFry*YopQZk5G0p+Ja6@T0`_A3fOD^q<|-<<#2oY@onDX?IE|16vg$hM z_7%2gMPi>7SFHQAwD?NJo{RaO)0&B??Zi?r0DWCHcA=#^o)0NV`fMd!@fN1?trK#)RTJe#fJ2j%c1BSIWP_*Q=Y$XYbx)qY z7NPKaoRoB*T<)h|Cw_7+xOTkEH<&XG1}#u?JA8oSh=E8mT4~Jl$si1IZ0+pq`06!+hi{3w7N^? zLW+u}K)Io%F=dL1N{UEgE{K2|q9Sk}o8_FQdCxh2{NCUBocHto;7KX z{rz5X$6b%ET)t^J2n1T`=y3Q)5a^3jAkebVFV%ogcta7DAdty+$HNCtM*0dlxM$rc z3>|6wrSnf^TBXf>9=q0l6JY;*rx8`|7OE0NJQL(omgwQ{Q{8uWerwwu`=I8xeNSH) zvXA*#B(%L7CU;*>zfz+6@POeD->g5l?)V1~UMS~Co>wyjeP$z}@0%^8g6QmI0UFvT zl}6*m&lyqE1K9amocZ=;AkZTZ|1cH6sH7FD8o+-6>vn@cSJK_efR8TNH+%sCJ=yr* z_XSz2ID#hQ*d3Fx%ynMl2i`Ya=j(9iZiX07Nk}*_G=hUF5gff-eDPbHHmau%6D=`4 zjh!E=BEpMLKYyBT0D5HRR(4^}UE7!3#TKq^w^T8piAy!Eo8!&=x;43yq9-cvz5PM_ z*f^p;3NA5S9JE#BipDl-+AdzI8hxBlWbp+E4JlD2-fPgmq|O%meT%#Ka=u@EmF|-_ zog+5V=UBGp$f=?v|HMiNX6!^4V*;j>z^aIZnAr-vi^1Z1bz&5-f`%$lZR@0fA>wD&GAu;X0w!+Z_nXPbq+WTyz}{SqIl2ThQ=1N4YG_HpMWCF~&ImHAbj$%X?nc5(&YlX*gH{~)Z zfq_#@ckLZ~XU2^QMUNE{ys1I7k_+5y6DoWU62~JSnY6k{ptt+$y%w7tbIJ^k<6Ukx zqE=Gq;(;aKnepoli~L`e4@QSs1z2az$3bEqr_1wGeADY@@=e_7=d~eXD5}eyf7i*l zaQKp>%aec-bW5sRa;)dxU+Xl{ma&BTkami?@gQsO;B zQq9x!9q{tsLKCmEqsrdVZ9L=~;TsfuyrQsIThkXOjlh|`7i?wuMhDaH z=7vSbxLB9*Clvz*iZk`W(~=JSKtV>xrc2eDxVe{hb$E25E7eoViGZQ1cODham<#Ea zce#jh*$lxJ8Y~+1FhuktCVQ0xeyc(+f{T$}k@YA__sI1*6oI6nvFGVC`OTF&VV*31J(1WfzDL4> z!JtYuP0k*Z-W}p1GUYurQK9ZNl#4w-W-W^s0*A^|K$%b7D|^-=;#3n&t;>yCzdVQR z-a~1!+A`rA!a1vteB&YR6UxQ)_u;vl*@@`~r=dp+4quwb6!hur?1ARHGKZKKoW ztN(&kH%ned)f;0>JRrs7lZ`&eHfoB}u*o58$$}rRu;H0JH`%7Bx=LSg2}?t6->!W< zIp)Jw`5X6Nj}*wEiCaXT7n4l`yljS5G_Biuwcw#=eXKbTn@)t4m(-T6IMO!~dCWK| zjZvw0j4#!B!$@bW>knmsOM)j+MU-Yirzdks2!L)H?H@R7A>g#!*&kARkvefV4 zj4>^qqv6Fb=jFqM%UijN!)ENdG38JNjI%~oQBxw9>OjE5La!QoV?@HWMY$M9v()J1 zqg=CMq}AM#x&zs7cDsE{reU_tWaC;}oBAz{x`VUG11mvoR_Fr1U7TzZmFBz6=jl(5 zWQf7LDUA*`-#Ker# z``F3LKpUdM0icYzxvlj{Wivd3ch#nu9%!;*e)kWu$!a}QJbBl1L=_muGCeqteU7brtt({@jFb;*zyfcY@u7jv4cb__z zeW)3N59zIRjL_@w<05w22Gs@gj~-qMh7i6Q*fY{w4^3uUkF>V{l+3q z#Glzzoa#Srxa$|tl>z3hYY#Mz)nN0P)qRR-TV(}`9m(>#C>sIJ!`OcCWGRiN5YY55 zEk^E16d>(+R@`}46x%%9TAoYo5^yLZ0_0osmusb_{i+WcUpEaE&tQiM_j%C3ZI=h) zOpS}!b4u0VnyZ}>rCGH)}m2luub=7rhhm)8)xgb$dkR= zOIzq7d9ZnUw_?|j)+Vr}QjhgApij_i-!@tvqBit)*&=w@4uIXM4Qe%yBch2IRUr=3cAdM_9Bpkm0(yhAs2|n zQS(N5VujX1>cO?I~!I>{LlzZ8&D~tqtW=_Tv zHVx{f3AYtP-x}J45}6r5RJV>i(Jn4{Z8Dxjh`m{KV#|iFCO4HmGng#@VnR|Y#*ote zGaXexGg`|%wCC-<9o&_>10VC$GIt&6`Wq8$GGvWGk?*ckK;RVy;rRKt@G$Mf#*Jl> zNGmuA^hkQVtm+m)Ep$sqBhy+azGc{DsA_oFD{;+6+zZWVl`H-B4P)Q<64xmWQzKnm zutv@`qMeEz^+=R*`kdiH?Yrz0kTC2 zBY-u#@JMi@A{jI8-!-%dT7w(@Ao$ppwI}%A$BIU<#!J0kDJ7P!AHBPF$Oz%cBtRe< zTDu*rB19L3O!d4FI)WAG(_M$4C+YE5A|mb3K;xIW)GN-irHyMLdDL2Aa2EzG@ZqM1 z3wyhIwW+eUSeyI4)9I{A3R&E2GZ_cTTpxLOME<`HsYG;4o(CY3Jt-5 z+M82OUv@-Kz1~42Xb&FTRlU?}tJiRh8^oI6muN4mGDxXPbnjG{wTqQg;Q|tYM&>qcbf=GEc~Bgf}nu zJ;P2mX%oQFvn%GZR)%Lsl48DU|Uf=aOv}&ZDB3 z1upVgInP9KRxoa#Ss2}9=-Up&l}!ou5umHzX>Xg102)&txTJUG(6g*k_;o$PDj;%B zadsTD0F}91znt>QA67nG6FSoB!loif+4^y|c8l|4p0uj(4kHh24#I)>;a((L?~S;I~BnD_v+Le{C?2x&vm5Kodoac ziyXWyr*JR5T-h}{H+r*JzUl?>RWzP3n`|xJorDyWP(4T~ zacq7ljD|X@4|A(h0ZLvqbdQbk}tZ#)>iGXub;q zeL`>*vy%?)l;r4+TnhX=OO9RSRFS_k6_Rq6Bf%890K9o8WWaa6 z+NQz1m1DcTu$#XBp#3#%;%5_Q{6Z(8N}mSpD>yM#x0`k%xVzZPy*wcTb8fP8WGT@I zc;!4ELc$1xn83Y&+e@%@s&0Pt!F1b^+n^jaT&a(VZxkj;$Sk^Llt3EW3IX826zjjz^r zjQ3P9f&MCa%myGD?i`P9s8T9qOef!|yZZv(*4vq!D%)Igs{9*a-*ZzVuczO(;BA#p zL;ypv;VTuNw$#$#HpQK(9CW@8j!JfxZp*$>Iestq+1<4d zcR$B(8EB7lWtq9!xt#!UwPLN2Vba0h9y;6{?|=N}HC0gkp7y7_4SzgZ{MSdXxFs{T zzqr&sZ}D*rd$3K$moJaNM?U^ygJfn|#w)A?A>YC@Wq}nzRE5C_XkIY|BGJ>=9j9^5<+cj;Po%U9s0!2loL6oVSm~@lNU;w z-R)_CR1VmUjQr`@sAJV5Jrx5>t3NF~1nE!}ZkTK9vG~(L^j+ig^!nJ=)<5IZb}#Ip zm6}ah_@93MoSmo9yLTd|BJ_X0@>^(I(pAl@tl!<;*nMq(z}r;pMzfI9L0q6HtB*4d zTzvOS0R*4D-8uD_07iCJcPZ4TQnT*U)dNmxcvM4h3rzmw1mg=(82N&`yFhSs}mk1jGdbmYfV?KRwsnaV`bg9Zobz zFg*#BMxrsI@7dm#OpA*11~L3=;VWWB$A90_x3|_6FL7hg(3bsjN=ufNSxcJf4wtm= zw`Cz;Z3NDVO9|%Qc9&xq45n+SHd01_3Gx;h^^n7G&7`sZ> z{Fm*J?7k`+)L3UFf?GFsIWs@L!eg84WX&&F`GTZzGZ2Mb+qHZS;L6h979f3dZBN`= z1I{r9*5V!E8+w*@w#Xa3iU*Tz>rfb=Fa$O~H0l~^$uJ{qG!0nycV4VFIh0HQb?qW# z{urEm86^`^2}csN$g{u0N(~x~Q zbmxJ?40im~gPo`TJ5S8NlSvnTxt@%*kYI%Kg<=YPZb&JL=ua`Vl|-1q<(Jr<8w`W)BcK+o0e+Rzo)^b#i~Jq?vlW_se9#j zgWr{HalzYwT~haKX~_m#l|ZX!Z%J<-_&nHe%e0E!i5~f(trkbkJhgV8`o*L8)SYlq zU)sanUU!EXcwD2*Ugz_z18P9`@2FjnVTjba0DyMa5YR6bq?y9?C+phnKaccM!HE#{ z?E@~1`_8?&ANFb^JkKTL0D0CWecvuXk8n?siZ~hDT{9#9`8&Vbk@*XNizn7Uwi4Hv z0>DTj1C;_<(yXqj@9a-c0`oArw?EH6AUZ_ORTcG)53ld}un4YM=|J!42~9F}Lk5gF zNDCwH00=G6`RJ%^0+2SNF%DvT04U{Y4hVaoIgqEN-yi)7wXyu#dU$z~tzxcQ;#Col zZ{$NcnPP_M&jbo7@}FD1gwEjo8-~Va*$7C2meB>FPfaA!yDlcGw>CP$8@R)IKf5%r zTC5ZNdE%#?F*@Z|>I#;0hP~ZO&#PeASP9>i=aLbeDpX)^Wnkdq++>%P%HVWKQW*Dq z?&oR0BB$wfko(W zo=j8?0O9`*|0(DH88}!D-9OP$16tCHc5|XevI6^S00M0A)$JgtP2_~%Cads^ z?SEga?p#ALz|y>FhAqsDs%w5}vrD%1a};K*voMfb2V5=8oz}xkj?ETEZu!3~9kbWg zLsn)3g9Yvz$F{eg)x#}LmIx{WT8{ZpE*2WoS&Y=Zxqt5ufZ*}Tf#$OD3ZnKiVr1+Y z>;F?NIrC4P>b{2RaNL2C7ZZ1S60*@3EQeG5i`Gzp!r5K-3C*x6kx}Xu7#oXzoX}CK-!IsA1&@2CrF@JF@&g|Q_07V6!PGb}+YQ%LiZ<*-XVA4lf z&p#O)UT^c>A!f)B=IH-wMoBPHU~8uykeumy5vS1-A}mWk^8Hn;>}t+C8V-jZvEq8k zG4W<+*>o%J3ls;%%BLIPte3{A-0!?`wHUfzd7-Ol=EFTmfnn*h zZ?d9>(}H1N>DE=WS_ajtus6ap?|6`Wv$BD)t{Fg(-y-iYk^e}O*B?KPk#zB2XqY$y z5~?F!);CggH{VhYK3uT}I&%YB9qtZ)Y8i*i!=7dRW z<`=Ps%VofI3Ki-^rNy4tDo9wZt$F^V!p9N#5~{zO{7P;T-P0Uqy5}ck_vY-gfB;0v)H{6Pkys88&H+Z^6>L%`1TflcC^TG1dpo=QM-~}aQ6ugUHt6Vg`fvC3{2?-= zvHS)cT7WXYAT>N3sCa|MouS=>=7jB>_a0Vvtf&~i`ek=Pv zCe~_Wd72k!i_Nstz&KMx$*Z*DxnI@ZUjh`6JhU`)k$xsK3&+gp&oe6={UaqAB#RQ#e$^axXr>+jC0)|`)46$BC64ZccRo(vp;J35Ku=Y98z~q?`RjutO73I&ME35P1H@@-ojmqyYUSo0rVPYvv z$&Ca9Qf!I9t8aU2JQ*hb(0G*fT8l%+hk?8@p6-_sw*Ao;aR!?h*BZZSM(x6B-LgNAP#*$+Ip zk>MbdiV;voLTn zXYzjDk^M#%ho6h}vv#GVB5YSM{C$v1Agf0uB~<$$hV{Cl$uC z&Ycs&fc||X-%9i>7X{9Yz+=jh@!-F{mGHpNDCIHf#|>h)vb%xt)h!zviim zfJ#nk{TefI-C*4FmsYT}Z!DB}s3zeOK%4O`7*Vb$iXrn*KC8k!T0)`LmY_%819QU} z!z~#pfP&fEi>Dd;`gCFO(*Q$8C9rBmsKAaq$H3X~9)5qcI2dDeWmvFi0*r!#R_Xj* gz58#T2v=&{`Qk{$9S&kWU=hgii0k3fLw?u(2TJ$Q(f|Me literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/huggingface/img/llama_mem_alloc.png b/end2end/liger_examples/huggingface/img/llama_mem_alloc.png new file mode 100644 index 0000000000000000000000000000000000000000..8f89581e5c0c7f2838aa1a9a8bedb05789fe3e18 GIT binary patch literal 15422 zcmeHucT`i&_I5xNP$?EfkrEI=L_m=uU?7M{2c`Gk5fBnU2vt-Rl-_&qH58?Uq9W2l z?^Tgrr3C5UM8W&M@BRM!e&1Smt>4wPG&z|$GiPS+XP*7+eO^D5mmoVya}ol9kV#42 zSA;+a)gcf91jz|-<=doF3HXlyp(t?=l8>dG1z#STs7sm3$w97z&m<5+g0qlg_$J^l z#K!_c^y@PO!UFz>K#nC6{Oi|5!lPda)f0~$eI`KQ8@~Ni3j;UNuuxV{g=8rfQN z8W`Iens7d~eu8fRx%*TQe6%(}8bF^~Kej;#K7}#;Y9R<+%cn9PyLCxTpDPEJmoPPaL2?ajDu2?z*q z-Q?!t=H>t`I1tV@NQ0*wHi#>~Z*p{>`z8n@dy6MX3tJl~{=No=whl-b6BB-*KYxDD z6KP@kuaRsJhmQpwkPH6{*DcPQTz~Ejy57ZK6;!adFaaau?=O7o?yr`AyLLFvT`v6K zf0@khC;fF5JgV@?yIg;kP59&`-LJ$Dh#8y4u~$V! z#lC^g>5JDfVrEuW3HO7YF&P^h8;q+_!KF&w<>x%MnORvu0m9poCr)1s3}#1VEi)Ya zl$h^8@jHw@@KVFOF26&^mn*-<>%*ztLAA6IL}m6 zM=>~m4??3hZOl;VTr&qUU~x)f-M42u^R(W*r*)5vKJaK|kXV~)4Y?{4{>7>Q!=k3= zR*D!|8maW#_xRCryX^}Z|1fk=OYR#(Ucv#i&B$P9US0A%(*Ge6cr!-WeGE z!P{L)V#JmF3%hG=1!{dwP12~Jvh%qFaabi9A;-spcGHE_2-TD#W5nH(QgS=gT+?ztU(7WF#0M;w;-H)rfFv6h*d)Y4{n4>srS;uMQ zDi>xw*e$_g{O05Sn;p*Lnmz)Ess7CuC)x+Eq-=)ljLMXaw3OOZF7i3AcSxk==F0jD zOkM2Qp4ltdDMq8`l+D1}*l&(F$xLJ8Ci`mk9pc=Us4x^82O%Fm+?(x4@j3^@2&rG^ZnqX6k*Z4C%nm$|c%-4}Liq!+4#{SGF@zV(gXh&cf~D zMtXVIiMRas4>aTNI-}QKu5Zs5^u-jU5Da{HacU7#w;m{Psr*7qyKl7ygU9XV$QvWo$o$tEtUS{Q>VVy9o6$GF8*{fJ)*duUap1L;C#IkeC8mE0FEOi4 zb(rsM6r+GK^N+a}(>)csT_z>KQMtX4BaE%-*y?CEN2(^lqSU6kbJjqRU`$ML@R z*qV%3zBtrJzRweQcQ!TT8~w)zcb#(D5#V#;%57F1}vb7rqKsKe@rZ@#Z4#MejYq#Iv1V$8mAgHO6AlRrrRQO*&Jm) zGE{11)|Mi)J(IGCwt{zrreXvIiPGxU_xE-dlhoaMHp;Are6c@XM^uFl#1;@VZS?8c z65kb!kUW+}t@~6Ii4C^tv$CLw7_@@g5Z>TgG|f*thwZ5Z0Ec72h?M40b7(4a)pC1i zGoLJ!g)ats5l~0L6bqk@jzA2T-*e882f=*IYpLxZ zgj`pK0)Nb0RkVTl<^wn3VDg%o@+mnDAMS5y1I1=TFoOz(>ETZ(ca^F&WDIE{Qr=%^ z_DPcbOwZnueutZEm5EYBKOH>Bx7^G?nx!$ zkcAp9)_j4iu zwau=9B3^G*PWYI1uDJAF{rUr5M;)?pWTs+Bb=D`D3cazJ4X>z1X1eW8PnJQ+oP^9`j6oY9?qya;|hjl}>sC?wjy3uqBscuH{MVR@*m8GeQ z^v$MR{`KsJtC#g>+e67sEDJN%+-12bI*`PW&*B061d#%xdxPiCZEjUO&JUBlY<{V9 zx5%M=Bb_B~C)M_wR@vhspP&tug0B9DkrCW6_VuHeO;O2ez-m{0ojB;2x)ro7Z3Ep2 zm8Q?kRb5}~aM(yhQHSTa4~Pg}iZ}u?xk>PAes*>=Y@Uzpj28XSP3>An1?{V~`q49EMRX8PJ6URI|-I3CGc!cPflT#X}IX3ppn& zP4heC_(zN>nk@oiccwpIwk=WUO3HirlGSL9mm!aTxsxvr_aUNWMU_(IzNO3Asfy+cO>iHFPyd`{<>ZZuzp?ql z)JB&ZX1BDa{s25=@-@m=jQ8S(l5DeSQfD&uhNXO7x139Ls3b3#k(k=Zxwp>BHyh(A zAM782jGRU{a~3CjEqVIGaOaZh68)7;fhi@UZHxNP!QncnNMvMT(bhc{3YhL%F9NxW z5t@xjTH?2uX-~pw)5T3Yw81tbTXF4yJ1MKWvdCA`2`M~?DOuE=URP&y1YdjY1Chl{ z(XdJk&kA?K0^;PnK^@VtZPWb5T>j7(5#0Qt66hgJi59o>oLd?NUk|++*>(t-^m<6f z_SchS*kID8HwNhRCG7j18AT0qlttL9etnfxs2lSjyoGlkN9UG{@jfbG-VlSD6At=q1$gX!B+p$x z72H}?X55s8c(-0xLgUIx^FWS8M8v;ibefpKwcUMX;gVVGSBdcNUf}S4t_h`tsV7}X zT_Jf&*)H&GDko^2XR&eptoE$HpkM0r%uT0}p&stEAYs&vo7RIo z8QV0>Q3c=Z_*jY(Kfd1Pm6?h;({ipmcH^#uHDc;b@d+`*vnnt+fO#F1fF#FbL>{b#+dD-NrvE zB~hd|)pOOv(}!U_B^!ZDAyZGR@(Ex8_?Y7v6z=(t286kiysTmPV;8jWf|5|`A5WsX zcI>x|KyqK&6T+_8DrNGkJ0OH4ghcA1=Es|kgodEA&{|(*gyN;&15lDYgAk}vYI<=W z>ILvrB9i~tDL{o3OOv=s?n@=6Yg*!LWwRe>IAg@onpYhjr_&)bAEZ8hx1n%0YbQv< z{=Vv2VtW!W`9L_BI00ql8FMzTxlL?f^r}Kgg{$h2d#B|}gA`qi1+k8XBIA4+y>#}V zTLm2>gvilLovN9Kfk9I7 zlCC3P+4wzlUD$<=<|5tf?|gP#s77z2GjOOjSfj6&T+)8Wqn78VvE zZLQDG{b+oyn(j~X3WzGy&_G5)F(2|UdpQ|(%AuFwRGPrqQ^aiqXvwlhthq8A${ zGzpv}Fgcki>ZZZ7g~8Hh;q)SpVHgiSl1|wD&T79so6(#SAZ)4E6*hB)jul$I>pUIj zw6M*kSt7Z*wr2A6!wcN@W!oyo=~#q6W?`u=TG*{3)(tn1Jj7G@erKt2(I|LtI=^x$ z`0GcXp}xJDlz_QdKBLd4%V*PLzE1>*goB7seILXPeFI}zO03RH<V5xNh!T%y8Cp%jju;nWmvG%@h z!@FB+i}aa*!#9~k=&%%2Z)dYMK=_E#euz?wPb$3FQ$8XOV9YSiE# zhDS8){w<@=UxFm)HO#aqRfFwh%VHm`OO-9r`T&d|DNG*y&`sg>kx=aA=^~#4o>!Vh zhA(8}1XYB$zMtzz5~RD(y(fr#+9|~0vJYYh^qPWWB>#h-vKVC~8^C(Q#K#Ohcp26{^DRzvRMKMk}81d9GQQh+E2& zK3CS#dM^40pU3p zCEox@GufMFL-L7ahG@~YX0Mkt^d;G4)6Iu*2&@1uXwci?^Ch!8W+N{1d8^<2xSA|7 z;#f{yxDhzEx8#JaHokxy4a4qq8t}&1_T=jbO_z-APVpz78!4p*6jX;?M*4f?V_5QB z$m2Z$a<6NrjC<7Cny6B3NS>>VkMGhdh_?wJcxFs=Ij?}&ak8j^zL>R9WN*D|wyGrd zpk^{YpZ}n&EZ%=}Vmy5PN6Ic9e2ykhXQr z?es_Xf+|CbTjUpS!k2(l#uEWIRv%Mct>ht4Av3*usDcZe z+Sha{9q~Zh9C^#=KB|-_%`hSE*1^9faA5^QY}KA9iq@|q9H{ZAsRB_t?qOndQ3;)u$PRwX?Ic@!IbN4ozknq(u`J z8}3a^G1xq4mpQRS29ah9tOAw+BrZ&|YW7hW31rNl{ zK<|+vr@f<3tV`oLdox)otKV#uuONosHe4=-FG@uhP=()fkVsT3&{cf+a6FgatZoxx z)swA)BXV4f!~5x*%lufj1oBgJM4Lb|+bMNg9%UPuHg3xmve6$ ze+i_q`q}2*;}DWNt=oe~U*6u~+~{hzonD|ie^|(tsuyNOTsX8|h!@YvpTdo>xH`r@ zy#D&@^0RB>JCp{L>{3yGS|LwT>Zd8iSTP=Y$pgSpOzz)Sk~*A(WCkD{f~Dmb4!s2A zJP1@D@Z1i&e@JQmdjp8q|A#d&HoMP}#Y{-@lk;J*TfTBZG!xn*W!X97?#HUp&bjZz z%sI_7p?bcj04+5=iAU$#Ir4dD;E4i+#ME@GiseZH8YK(mc>%!-NqS|EFTEE83Yn@W z#LU>M+J3IGe8-DBS#rKGygL2I)mKAbo*#YAU7Wm?xJ z9{;%z59|PTmhX>)w}Kv)@eF2%a0~Dr;O|^e;U7OX$M;yt1ne3WAR`OPIf1Yik| zCK33+E0>0S%Nb+v@nvI6)a^z3?0(nMO-sN4J4>z9G-}*2Z!X<_P@*QIbRAc(n*cCl zMFD@z9&lSF#iH9p+V81Np-G^W4g_PULgBr2r6_*e_z^!|^LFSK$Q{vs#OE$5ZDxKW zDCa-ndmQCiYSqU%P-Nuuc=-Jz8_b4k`Tq8N+?=wqvZSM9874j^hCM7S?4PqH81o@n zTU(oqv8@gwrxTP^Q3*pXkLoxs4O^&XX}z;STXeBBHZ}(B?Q9N-y+Gj>qz0I)db0*iY}eaCuQf0_08!kYjgPIC3v*T@M;N>bqkx|r0&#M%g^Zb#OwU=?c?e9CX6 z(rJjGH%ZKgnVr44QOD=F!zv0y0(*m%PRpT72yT(8M05AL`l(hB_OU#9;{MYrH^Gzz zk6T%kD{ik=fKnR78vpdeQd0D7GygaAutBy>{P~Dm7*N#J<-B_JD)-YBCl7)Q!vuuE z-(m88=2~uk`4mmZ_*}P=LKq%zOPb}nI@4rKhTq%aSV;nemhL!zeL;=qibrQm;8_A; z@yN5P+~N?gS0uoy#6P{T1nx)(Oppk^07S+&7r?x3rtoInJ+cISV6S$U|M<**Mw%GT zN7Qj9b3j-vTTSlFK+P@Ju%>yAJb8CGy5bGF-T>RJ-?P6(xUOIC`P{j5yEyCX>HPB} zCr>V}`jit~6Z-wT51wBk*_CZboUA|S?mwAVa{1+?4B)!!Cua5|DVrdy1CsC8KxBu{1xDR6dWn#| zjuVI~J`zJnQXfID5~x;%$M6nJ_Iz#Ko=m`%Ya$T2S$q&UA_fHR z{#lq`F4~D_n@N=mOQY-~!cWHG8pWn>cPPyS{@Dtjv3R>Nu6X9$xY61ftB2X%z!r@o z;hH`qw_C#J_{2N|TKSI!DE;E&n%elNEyk1Z)4BG5I3Ix*m|cUij0OA#QUdz$to>x@ z5#wOG3S#7}sZ{PmK0%4sLZBstCWm~&OA=VKc(y>+=|gxX5(N<;Ngn*tA;Lfv0qF^{ zl)sLj5FWHeDWSC|@3B8TqD8Mlf%&IKhcq1``hTbJ3t|4d5x>n51nZ-2n>v?n>U)-0 zbUnZ)oAD(-N^N26YSND$1%02Kl(Mrcx}kw){=ZN*UV{%rTOS$rq4=1)$f%*|dt*r4 z+`psb1L@7bV`S|nLJE)~Ke z^Qyl`+63X13xj?Rpq@Y|Mp{ND4XArKm$rDA$w-C6Kl!0o=n3@zk^T3wF?_9A^5Wtz zjLpo3=F7{=6*V-X{;5A?zI9Z5Qa9NgF|=05Suw-Jy7_mUTzI`=YIYqLRnLk?=zGH0HQgA$`F_%fD(t9UH` zYXE_#;(hr#L;636$!V`y_2#~=6rtMrru*vcA-H+MeUF1fOWMx4k^U&aBB=uKG#>Mc z@DRP>{Qx09^csdzAGRRm0toq&CE4=`A^8A87M5O|#HT0$2zu3mI>zJGudhK55%Rel zK*+}&51bFR5$Fm)NH1HaldldD(f~w(C$;<@=^he<|I8_%KdS0C`QfJ=SV!nX?n~Rx zu2`;=Iu$568pKrItB^DN`BAis?nF77+Uo@1r!=Vs#J`YDpNI!OxRE9~%M=uslLxWd z`3lJi`4ZINdp`DRXIg@!<9vlU&SligjVWfIT@fd0Ej-}Ngn8+=erKZ6)hV~gU*j6u z)@rc~E8crc?EY${yVGfzOJEbie!K$($#`;hyd=Z2nL9|q%+Ga?A!xOgNy512!uG?r zl2$pyIr^cJ2khc50Y5pvST;Emv)}J+quTl&Ke($^diK(K4U>+-R({0e=CeU*UH5UW zI;a-X330)=`<^N}bv1Z;{+{aB$@Yw+aGpJ=jh3mq_Wn7hPK3U-TeM4jhk068Xdc9-?L?5THPfQYYh^W!`OD%sEsR=4 z&EUy>Ns>=zK(fmF1IsVd@gz}RrXE;H@5x!W&FQ-?@x4F6*L691kp2o;p`%t^TXsrr z0jVYbljm|n6?iTC5lC|L;n~`Ei>o%7lOpN8R8)PLo&Ax89LWm73Bc%lVBjW6D zJFGjOxvOft!>S53+MsbLdzLZ5RXU})Q9xFP*3_s|W(Yc$YYls;fsvkM8WPTim1Rwr zQw?f(Mei?^PPxx+@LIsr2-GSkEIlmWt#SdOVy5m$>+q^2QC1Hb*tE{#kY}|zi)xsd zr_S@HT0oYtL|bM5On1Da;9Za&+n4WTExc`e8SF_IZzBe&V(pnG7D!`mQQNasvg~g} z>U9S=hTX&dU&%^JO0T`%iHl&R)T37gdxgW392YA+5QFMNLf!JN1tl+6)@jM!MB9%# zm1{bG*TN{j=*Z34UvtVl0bp7{ym?rM#*FL5(O_&Ca`J{S+X7{2X>zn8BV}w4- zgC}t4OTT?Q{JeP{Mzxpq81HOxK?yY*->kR9rqHYKhd~45L|yq!OjMp-Cp8T%R|e9gSwaMQU^& zY5DLS?_JBR=o1dfW}B{;E-78@`U;P8Ds?unPD~?+Ag2qaG{5gDp#GasQ^FHp97m3- zydzNM>axHGzp=!QJL1ZG1reb;K7L#=z)c-l4_(Xrb`?}BxR00Qj8-K)x37qW5)CBPhV>2-LP(L=p>GtOJUsJOt^Y}eU5D~rfl=xA#Sota7YU<`%+9F6=>wffjvW373U07IF@1bpm+bhia&!DOnh|P zE+eJw-Am>?hO>lrL^SWOOwcy|@^%6uO-(o|4f(Bn|EZK5tgV^%&IN)9Plt@)Ln;l< zlil;s+Et~Hsy|m`K&U@-6xjcdhWhsy^z*dK6@7hunYp+WeEs}P8iTKvTt7U-pzX%*xN2@;^ zOU+56oqyPjR5Yf`#o$vLxaBb}d@cos!hLGpNb;dSyT^E_&O-=`2-S2P!e#;(*wzVp zK`Hrlkd_PsiHCtg18?mob;r?`J=aAZe>wJNj9~c8wEo+Rx9*v?#dAzfPAV!YGJ%YV z%~>MWKR0zJeO^s$1=8W%Hp6%DY9y`Gu-)p1Q(Te<`+IwEqVXOm#%=N>IED|JH*f;PMV|0;L>F$TLp3xs9I)LeTTt)Q3DT7v3M>UWm1S6W zRmv5nhdMeON*VH{%Nd|Z|IsK&c)DaXR8q{-eR}y24E3U7>w^v`Yi$^!Kn|fz{Pv{U zNzbv|q3WCz+5Jj^cH42hB;=SY8-I6Tpn;IE^-LsQjp*9gk+lVN)A+3O@Y9L6!?qjG z`^Uw{y@=@xYd++GO6n!>R>1pBrK+8FVC;N930-|}{;gfHQa51Uz zVUl)kF4a9god*F`G1e6`=e5V$m#TMc%2z&|noyM<`za67=I?j)#U>p8+RMNgsS#ca}D%p9d5lToQJDudJdn zFlJR@-jUodrY-@s@B-%A2xJvw@MZ;0tz5@rZv&rl9-QWPWMFY4AV2*A{}|@!GZ(;W z<8W0Gpuko(AA=iop~4qsz>tUzaq6C-h7&|pBs{%VPevU28e;_z3ygEq@hEJ(&*W8G zfALz5K%i2CD)35Jqt0|-^%_+I}l zK$otrDRF!|;?t1{E4>Ls&r77bU7Qa=X5bVxXNix=6gE!FY~}$&*AuX!&2F0`5>LW# z;jgET^G0%uXo`WtTr@UBqv~nVh&R3<`3xi2-Rn;puSR?U=E*3Sp@4cI5kS9aY_cSZ zIEYf4hFoB}0;f5kSy&;C3tVFB^+oAZ_iv20NM9RP<;^ujE-;OWNdUJ;kLz$Ret$r@_c#N zhO#p#a83hGwjpV_{LzKDkg*#ahDkQUCj$U{f;)Qq%ay3oh^^-K~p)!zxpe~=w&M^jWE@9n& zB`pfTw&~X20YTn9wueeeSGJ3d8pzAnJ7r9EfBuk%an|hGUjljQ#{+0_@Fvey@Jirn zj^19=KpP&A$5o)s+b@(RHkkb-zy^I_PQ<7!={e9CZ_IW{fPw-OwJRbbdSx~v*FjZ5 zao;95iZCu4!JzQRCo4J>oLq#=U`z3(b0Bo0A#n0ctLwriv`&rKH*?@}aOPC!&woi2 z^#;jm#?E)PW8LSUYlDg{Bfwf)BGc28NsP-pqr7`69IVQLn0b|-zmI^8jV)+y&a(Vq zeeWvA9#sg`)bFKzy2Y&Mi16^<96xdf;h*E) zs~}|qitt?(ZAI{DJ}+ZHoPr;HyGR5K)KjRo(|2L$odRC(1$m^h`JQrw*%o*cB~C!S z0t5U|@zJ9*b1q6xi73ql2wswCVdL4ac*0MS%+Ti6>`roVyZ(GKJr|+l7&Q#a2+%eo zf;&ntEAE0|-%JFr8hfe{5Z)*-qy zIC<)n33y+I8-;uPr{hZ!jIZ61=|e$99sTdT(V_QLfy}%!)dZN?Z@m)g1>B7g>iFlN z)dlWW_4hmcuF3n|9zyUhQ~7-hNl=~`26;7d=#GD#{O?Q5K;M*xe6l$jvNi|2srYLH z4(D_hR0mvz81f&Hq;XI))0}g$;m@4@%3AT%4s4<4q{^F#qgo0tB4ECxp8bCk%}=QS oE4=>s$I+brl|K6)T!sS%s(U?cc8eAGH-#Zm;_~yaB&5z?2Y(6EK7qrZK{+^_oSfL5xY_OOEjTy@1Ozzla&d5R zv4Ik7Fc(|6ku#eujP7?O7xhS(!A$HepTRBdY-v#S8X4O;z=dgPQ4Rh1_uEdmrTITv zvW1;@3v`eJ^$iCn`(2Jdb%R%hP*0)C_LgR#WmNqloI+wmih269bS;{Ic2^wkMGxY}TJ(@&1DOPGW{LPW3C?lX9CT;WQ( z9qFf%{hGU6&W{)Cg=BcN(rUAs#@O?BTKS~ff!wS%fo91niFw+TSmi^;#XepELL?j4 znqCs=p z`XSbbsU)v0OY-vUFVqi1Oq`-BLP9Y#bWHHWn;wtQkbyk>`^5uv2y4OFH>e*pZ4gXN zS?u(CsG_I`UTA?tB2iZzIsufrs+GydA;bEVXR_a5vC6V#4pN&X0b0? z`MBp5M3vfcUvbe|9D%5>+naRba^9GJxvyTJ%Y1|1HauH3 zGsa|0B~sW0%4SXZ{dm0Asc_tB3ei_?rBOYf;PzdZm0{mwuXd^}>Hdou>XUEODk>tc zXi>A~%+iDtSve^yI)*_YM58S9Q{uy{y1IHh9!DLG!SRyDgo%z$q>fk^;7pr zm-+Pixlw{1(bKvojc?C}*oWioz_mQh)Q9&edU1@MPL9_-m8(WuLa3vd65UnUt7nK7 zdsF?~+^yQ}39e3mh-A}I$Ui1D+*uh>WpDUp?6Q=dIWsnOQW57!ak@&}Bp+~x4p>%z zb{bGi5?z{9iOu`2X0x4zJG1u@+u&F-wdYZ>shL?D=-7>Ne&O6vYC*>p!d2?CbtqPZ z6$bSgBn@cxBj}($*|?t;of=_yI##Z7ui(lkjRx4WLMd&L`cP_(EJLQ0_N_mBQ-?(qv;-))+EJa)NV)~e@CS4YdkVq;mzbf0+#-QZUmt+ZPw`kWK8 z7Bp@fm0DT*>Z9hD-rQjWPrZZfMY?*|9eY?@PM$Mj0zya1^Uh^1+M?6FFj?HM-CE{Cg?X2S(~Df4Wstl>t@zR}BVT{EnYA3tsdRi|ZT$v_>JX4V4semO4}U=0rq zwX+n~(Jl|=BbSz*GPAQsUM6OIr@Z&E+h-`r4hqOEjDT4%ZWZo9uf_PZYFuY0J1U&jYKutU`s)cE0|M;qdm6oF}uk|mcg7oh7vW zMcmx9eMr7Ad+KP8f3toa(Npic+Hi7w7;AN!8Y04Vif&Z7<&Q!(LRR!COwEIq9l@#n z;JD|X$xEz9xLkGyHM(#@j(=Ec5!}3IDD}L?Y2B7Tx=H#)?$MEZ_5NJ!Y#gkc^Zii2 zvMAeupPV!goP;hkjn3%6?_tLgu$LWixO+XGiOCcUsO$@KIYO1DboEhxigoj-3=WVZ_AlJ&1T)PdQ*sP9!Sc@l^Qn6bWmojq!cW$ z3I@=`hb6xS48aqr82DoqX-?i>kMJm00~AKdw0%W(GT(1)&UQ$46c4|~wuoU=j4_qM z;w9|s=>EM#_<_IZ5adoo7XA8nCfslKzQ zM48NR3xz0{s!jZ`qV1vBN!nf=a}C|+uX+c(7@&)6$*t3Jd&>*287KG+o6umniZ96O zS2JW!X&7UyqqTh_(<|4vNx$(9>-Jke`dM+`PDkH$FKd5C9&tPU|mSMLlz`N0#_WmvQbqeQ28+pB9lH4BKs&ClHOp}&idoU@n z?obK~B&FU!FB_Y3Vfwdbzd-B|(W7q!HhjG`mK%AMq`c26m?wV_XCM0F-%$AZqa}pD z5|dY^OxbqynvR>t@xUp;lagao^1B`O{NP`4&|} zp`#h8LC`77fviW2U-}HW?5QR*6Fr{#u-2PdN)MC;Ifp2xtyNH3BqA#EGhFOTtFK9q zaX>@fo}JBvx~RnqEuP3&sOER9^;p6YgO|_|?1msG>`rl7a0jfZjt6vPdu5r#tOD3e zVe{R+S`$0YV}|p=BN#Tq4kGIE%<95A?bKF@Hc`QAA7izDs=zC;vv9YkT_{= zTgZHBm3JK4P|WDHj1x82OCvAsw}KP8`z@SVRYJOtXJqU}uQF=u<<&@PORZIQ6c>!O znyi$plEq(9etm#Ja#i6+SI;XYI8k1LMH18u{*A!Yb9 zw0ry-840QRvZu~Vs=<6Cu2Pp^HK&hv%FI3hA{2vQLxW5BG!?&frp|cG|wOn(Jux`1rBS_1&xh5N|dt} z98uXJFGZN?J=rk+O#}v8Z&`ED9vcq3HmwIA9-y(Q>kPq zycI{5wxNRRfG19=ufpJXGgI`qt&a5lS|zN>ntEil?bpDxgkAFz27%LrLlm?9e0Thu z>Y<+9Ps@jYxpZ)D*0E6*?h|F#B0{ZZ;_e_ms8K6@DO~hdsxa6#BKUnuKErx&&2aHX z5igro$;Tk=h1S>DYzJ3a6hmPND>zXN6!)d=;ghO)0&399di{Or@?HJ5F&fem@cR74 z^eBBUIu>nZ#ClW%ovi9iF7fyLOv+cKu+K_r{DOX6-S`2WQ@UFIzoa5FJD|xrV<~jp z74bEC^MaPHKR=PZ@otDQ=}8ba*Ly*5Y}+8w&*AYCkGN_9={9Phoe|G!M;-KKTHUUE z>4;@a-z~v8c(~4gK+o>Y(>^QJ7ee?++bjV*BLdduvmZGG2{^dE29Nik4kRzJyOQPv^q zL(R%NtHqfI1ERYJ!TGIep08J_O(v-9z#@+lk>z(<(*;0fS6v)#!hfm zfSECTzk#r=uXpZO4cAwyT0_;kO+ z;G|?9mj=Q1;zWWGVXum8s1R4(UcEm-+VeApBeZ6jxHwm%kdNc@AX+zLy>Kqa<2Op9 z95LH%YJX;o(3;IC7z8(zthzPO{PTapjwv0)3sLeWE86x`ENMeE2-PB542a#yX`bP!`hZoo)u45`a$><5e zS0|pgP_EfjY0mOiDoRr}{(NKi&ZCb;5mDZ=ZFA~K(mk2*CSkh3>@%!xPCjkTH69R` z<{+gd2`5C%BjF6?L@Jn4Th5-5_&|JbezngLn-83 zGPGY|#{Xy!lNXTI7;Bf%u!64R4fS!UdoC?FZyFp|I3vVFPMWos@vskcFR zod0@fQ^1pr0s`}ey+;*h9tPNby`}&bnxY^13Y4S{X@O2O;tWtD51JD{sm12i8HeUMKA-I>Q)pS1 z%-I!e{4OJ`+!NMc^1W4t6pZ9F32+fJ(Tw>ai$wP-HdV3?)$T+Yk~?CtcTbV3I$vDZr44oFmC-k`eTamA^%JAO`i@^v7T`Z4tcG-4`Fcy- z_lkg8h1B8U_TrBMY4+M1{Z)i^&(hTt=?}*WeV-{3>#~rD0$hBOpaafj+RsoI^ zWof~fLu1BZu0}1LW}@dwwfk1=RCrhzQ8brXfIGlOMrHX%TInadt7A+Choi$YVe;&u zz)y*RjhX;DNmDuD7|UFnm#1)*O6ZAGv^zMKBdiMRhbHEtUO7|7=GuKl#8~lVagMG}xPPO4N0kjhL%juiw*8ZJz6lrkgrmb20<&h5hl)a7UrXPOj7; z9xm<-ivN{$5wQG(epVg6d+@Q}c9Z7qWiV0EUpmUJp-WGrps()d=Nq3NV6ajOm!J=U z^VJrT2mGgz60<0g9C0B&}^kt z6X2BH0sd?O%Ahs{PCBaaPtDO-B_ES*jFr^q7ReJ zWdz>DkSGI>>gwan`vV6%%d^3PYj5|aJPo$h<0nZ6j$M~Gj<$ND#>aJG1Ksh0iqFzm zDCj{?mnAHJB335rb!y;?#$06Auk%8)a&qLlq06i=!HrJtp78eecB#Xjdq0AtmOZrm zTR10ce0!NDE06@h&F^U27_)c4w^OGEjwdAijqC8Z{(+~EV7_VKu3N+FfX! zQ0cXpFAf;#Ta=32=V06#B8p&Ir)y41Z~Cz{=V%~-U{x_ZWG zfd*D~nb0u(O#(4W<3RBiP1G74EiF4^kf4e_Mj&PJxhe&neT@b&x(){1s;7_ryakz9 zAcT!B2ymj_24!BVd7-i73Cq|1?=wI~)~noUT_0!yw#K_dI(BVfs2D^=4SWfx*<<-^ z?6+3F7K@jZlu#UhjlaQX{T4u4TSo^HC=y!bPa`H6*j8cO#lO1?40gW#b{oxVYp#pS zyz{T@tgI(}LCmTdtpHL@0Wah-`hXeDXA=`^v9mO2t^?$*uhg}$-IWn8ho#5aIXPiK zVQ7yNfUU4wS|-N&V_l4Ulejl4)4OctxqVzbyjj4Qv%@1IqQ$bSO9&{T^ulht;V&)` z#IOSCtT4`T)N&Jk7z541Dl7r_yGYZG&%)1A@WtL)j?*`X^d)};wR zJSs0QfBv_D`$Aam?EFA@5F9C;LV3VTt+gn~%Y#SxU8TI!5y=J{M&U8gvBHo@ItK>_ z^RT*!kv@^w7LO%~gVwFDtL|r#AF5;ZZhMg_DJf}U&QyjrQw@4^-SI47N~|i-fo{jI zd}Z{CtIk8Fs=dYj7I!I;nrPA|6K31+zSo_|4`PWRMXxR$&|`do<)|MD6JxaW@W6h zkX64<6C0lb-i~0zXguEsXy_qWDx`)k<5!jZ&bJcIFR7+}otF)kyw~qv^g-n|*lrJW z1EhXG@a`m`X=?BE$Q(T1zi%Ia{Y&y)h39-0O*6puGv5`=zj%QJ7wl)ksFZU^py3Al znX7cllK1?Dw1;4SzN#X5b8bX5GiWgSSSk17g;xMDm_MdJIbX@G3ZTKx6({YB7np%X za<|weF95?FG-zht*#Gnb5R!pKxX+m>&XsX4RnQ>!VZ8dq3%I}{mDlpU7d?tboy{Zb zWfw2}=R@GVLK3NX@jl_Ljop@(nJ5m*NWDu-bQRbp#nSVv)yp+P<8R z%w3({K5HKAC=_E6E34*a@+g&xyKR zq{61Ke_`r#U@H0WEYDx(eN$lrgZI>^{haH12N6#*e+XK(rvPZ#3lSQ*&h`NcxSpEV zwY&;kdm$i3>3cENLZ)8vprKCz;N35f&msg6#74Nyl%{8f4Ms5gzg z)S@R6sQM_0I*3wG9nh~IbF!aADxZ=5S>6O#VN`~Cz>na9&2_$0)GRiB1t3orP$bjO zLF6SK$FBSacXq2^mHrKR&w=c-wNLONTj6kNY3sDJ(H@-CP-TYJ(UrAdBPY_ zAu}38*9v_hZRT-rKbpK9-V)}X2;$&fPb(9?YB7g^#Czs+j~*3)-S<~Qn-j4z`zkVA zKK0o)iHA9SzE{d{#h~GErV|OI+t@ZH)TR_O2Y`N6UEZ2bIyf8)k5tCNp}aE@9B249 z=6>6AMeE+~H%io^^^8mUGH@zkfOXtq^+Nyj%6{XOE)@Q$Fiu-~@tU=hXjMH+URZ6) zOmLNJ*sEW!#+OHRNI@%Wf@&oayZ+?;Q0DI0YL+FD8DaZyaB#plc$3TYwe?6zdYj|K z+W-Wo@<*wQQ;g;lBq7)^y^4(_%H^It|IX$T*9kFLJAef(va+8XgR zL~H@T)`O`}JcQ?F*8%3asU5dt@$U0D?}nA@0I?3aC@;o9fUz_ zRe77VOHBRYtD|aCL4VyEtFVFXsXc!DZnY-4sOVvPN5_xRd2{1`0@3Li`r9E&+Io6y zAdLCadc3N*mmJgq1oa>)VeM_lZ-8RWf)EGGW^=*0W_1j{{}GL?kdUaFrd_QB-}_)Ni~z zF7nvs6&*;iarlxcq3Bv&V}rFn{iKpjBuoBIPSZ{#?%+@q`R;Q54wr~vnCF=L(5>v; z;CT?7xsHc%>8f>K3W#h4Drur(Unm!oQ7LX=j$FV#ts0S4NjCrQNg8^ObP?R=0F2{Q znm-;HI}h;`zs)E;5E_UOZ(b;>W+;TpA{pBNolmoyYAA93X{kNHF+Cr0;PBvzs zjL@!iw8_)sGHoRUT7dkALu0C@{{V(lvS_Z-{Az5;CfFqB$$ z>AXZAJ0J+V>`CXyMMKDka=&(NubvY=?=p~rkY10Mbpfw06~SR|EiC7D0k5k7UM<%l zcP>s%BS2Vibwc{6F6h>On*hf8zr;j%WugxSQwq|vu;liJT>n~R^ddYwTt;6%Av`j& zJx47U_=u|_m1uthDfL2qHeyDFw+9EVdo2|X|JJqqH?IxlX_-i68kgIB3b?MKuFj}e z0}mzVHg5%T(u8&X#5`%!KRJ3{^XNli4DxAK!v$Q%A8|%1Y+}a-0s?SL_4r9hN#$Hz zTw?dCv5GF%7|JJJ{Z=g@T|~;NNdr<2NbB|P!R8cPPWI;&uLHEA7pzT&R##W6=;$n~ z=a(OjLzGw-ZWaNwU+7BKYP^TZ)@zk^D+05DmuZaXSH?$cv|$~E$TFKNhdYscU(*ub zd5(oC#e9Ug?X9s%N=nKy10u!3RYeYkXR+#3blOPVzwdohz+S@M-af6MKxuPpYpkw^ zZTh263IiT{jkRbVHKL24{D}_bHu@!`eu1sdeUW>kHh~07VF7oZ)u%rw-!bz&cD1Sx z>It5i;!AXpS<3mb$o8St{HCbKxDe-)%0-*|@zVveq8BK}3%c;n&<~ETrK!fog_k>^ z_wDTwEM_3B!d6u}_@<>TwYr8z$I1Lclz)OgEXVB9+t#tp=lVV#@x-zK2O>Ho+m z0FE>1y39rnVnj3Z^Bo)Cn$7w%6bokmpwLA1|0LT&os(c~wj~I=nSOYAnagIBr!pHA z0UTReUjB6QHU3{cKocif`NMGq*7J1%gehES3d6%Gmbg0_Oe?#49$I??ZF|e;Z-7yDoBctT-sHm3mzlmC-kUW%}wFUwOmD#|c15|9Cttpz@qUP`n zy9CGb=Fc2Q8{NOz&m+M`mGDrx+;#%WO<;BO>*rcD3k%D{EGqY@#H?d{$HxaEU_bME zW@bi4PcJ^2-p;9_tY@w`@r-IYx@n4VT)i*O^Tk$%!P~B@tpc-wne#@|4k?ns=0>we zeL6nhBGcSdMH-T^xCuzr@{b-pQjyyhY8q`o)ON-TY5<>m*cv04TGTe-uQKxOiX^3H z>;0J_`>gEj6}6pg&2xaEDzV5UM>N4hP3hhAlQUOUi~=*5`nrJq!?ayk=)n(K zm(0K!sOYM_PJx0|h*kCDUD_{sfIpV2JW;fKQmhAW8=`Qdi>FH#b&l) z5&^z}ahwn7YY{*0Fu9J0iql4T5@)++CdaJ^g-&V&Y}ag?Rv*T#f6musw|}D1Tw%+g zl+Ymwg^Nb}r3aA#0r8hS8XAD}rg;cdr5yIA^)nXjSOQu0HRt}I7D=D&(5F|(qG?ILkXyfM>b!>1iz3MFF6GgZ|Ks7%_`8HxR-LrY{$s# z8(T;$zazGezol|7Em=5Uxcg~Yzh%1;nv-FzOqXq_QoUsGpulpX|4m1!Lii~s(=WPP zgyP`f!p4m~JC)pNpRE+N?heO4N*9HusteH0kF(nMuHw^4Q5dc|`6Lo_!#$`-M}FoY z3m>Ni!aD9&sxAiyT00(o@GZyCk-VzThp9>Ngc9DJ?VZ3lq4a>p*UNXF{`ZM@#Qd;U z!?fp6iRMQTg#*kd(}N!h%g5%$)X`dvGhDI`g~i*{OWjMA8>FqBvT7Y0{Xu&jD&QS+ zauYu1vAEA`DVC0|k=cMRim$W(MvNGQ$79|Wo%w}L{1D!5(_L}F8#$`e3dZZBg_eUg z_ueaMSfo^E#5D4cclS?&p?A1rdt2IbUXV_6HBNTzIyDcbrYoKb1{;1=jh;IfwLH#8 zAh85^fA}5bNyNG7sNeGJEBU9s@Zs`rPnLRw&;_&(d)}wB%|`ZAJ3P@H7%)H z5FN`|V0K(C>m1K}t>fR@QWWe(C+79Z?2MRB8=xaxGP_^(-OsLHme>{GAb&&kxsS|q zO>gRp?-A`h;<50JN!h`eA1Mq_=XKiA2e8zc9Rhq3RA>l zu;7zwA@93ys~9jmKqJ@_-&;bSup)-bDUEI zgZQIcVz)JXtoqhFs7Knjn>Fk5!~Sr%&pjdt8{k>xUd?A=F>fBc1*a>d1`H+3*z;zo zTXAFoP;_<02x1`t?{I9*LlKhw!)1Kp(VmkU;v?PuyX5-06EQ!wvp=<_!D^sTbe#s6 z{6pzBxP+o~VTqvBoH4mKBiLYyGGBiW1dDffcc8?fXLGd4Y!IytqI)#nwuhuP?A7V`*A3d87qjc({=Z&hGxJmmGn*?zSo{qiVH zzuV?PeBI=7iEm+x|BhMDL3#)DyP2w0aEGIQ*R!E2Ul(2NgkVjFXN=thadYliJvj4$ z>GAuT2Pvuu-j5>&mT^4^QOcM%fpW)=akh<4@fKFa5N%l=_}!sPzD|0J%jBDJ8jLgd z3<6==+@9S;!b%vKrJs3@ByzT6r|Tpq_x%+NC|1YA;2ttyfj^lG2nt0i{${mlI%3l4 z@bA?p^kLd=y0r5&9FS{}?g+W_7$(a9Lxn-Oc$qJvmW0m$Nv*na&T;?qOn+~JB0lS{ zGz<(3VNp>`>FMbuPX}}8i67nka{#>)o5sldSK&!k!E5}tkzi4UVL_-3^Z(9pCim}P zoFz@62=-ek@1{$d*RNmiE6?4sSysC9TkJ0FVl;XXV9jqQ>)WgC-(FTMyL|VB zypQvjcb+Q~)<}wU2$hPK7Bje1V*hKy2o*KFdG8sGIno2dCz`EMhmD_kN@HB z7MYAjr$-g{l1dlpA}qV9LU^M|iIlP|jHpzWwfMMf=fL0+Y&h z;nvo5TQxPc&CH05gtjUikqz93hX1i{kivGWKHI`(!O+(zIb zxa?hlvyk=P^JLyLz(dNz|5b3x)4PRq83#(j^`tvB0B=$w1S*SsY#;R7^7t?$jE^vr$O zk8ASzC!JGAM1%a}0_&n#ik|m+&{SkeW}6*NmlYhJDsYp$ff)IiQd zlLa}b51o&L1+RcCkp#r<$h$7*mEsoTlSe_((Qj~g=#o;gp?h-knv$6Uu({R<4Si-1 z1Dfw$=Sc}nI^Ny6lH}8&P2BRNWEqM%* zVcxlD`iCp(W71;T!(g+hjJxQW0+`Gx$ZM~u)v38Up}_j`ftP@0^1|n7{s;3Ki$b}r zE*uB56Ho*5$vPJR1Y}22eLiKl5~2cG5ye|YJYR;d`2j6^&M>_^>ceyU_`VHrN1+uY z95I97xXRGPzQNzk)~^3z z_nA_qJfgd* zK1!36iW54FY`C-xwoC+AvdXCYF#8kh9!wxeZr4nHj|YjhZMiUJ*EoTxL8kAVEOflx z-&gCj&Pd-oY=pdGjZ+3TYEo@T`*}FmiyEs+a-l^(S?PQy6Fvb7gqRa`#Dzv73IfFY z#Tr?}=X)Jf8xXlK6VkI6St}vnZ1Y&TzI1U<>wNb;ayf^ZjZ~mOP`kZ?UW8hb02Dwb z5RWf(*dq`$pZ$N>gx4e}ES|a30fty?6r5~*Qa%3H#AqVL)YT)uCW1%`N;#|B>J}_6 z;Tv%TseAk*r1keZf|(FUKU=tL$L}qEElx$<3fauJL&$@Cy#*=?JApNH;Jb@Sf#{mq zW2b&iOrKj14GcIAO6{gZszH978Sw9(s~v$Piq|X#LeVE6Crb-Na_Iz})=1d+L9n6W z^mx72@9D7&$}AX|#ICj?N{}(c{x&g@qjJil7ToQra?q-_PuB&h$bBHYY7qn%gbo%` z%@Zjfl_UazD-@5tO3Zb4g_)z_SBhI5yU@>%#C`Q{PHi9-f=aCG6{#(8T-Aoz3Wr4g z1c`ep2>^_0$AU^00}?nkGxcg68GAs`U^ek&F%hM1GRg=&oAz>J1+gST4{z4_#t*-G ziZ=GR*(7N0LMZaSdPZKQ7GVWg!s~DfZnGJRT)KMm_UVqnDFcXbL`|KZ?Ae1V1#U6Ub-9P zUe1(suydp9ot_-iImITm5B0D8T0cFp6L4I4chur};%zST@l2k-nyF_d zoTefYLH;8jOCqq@9_*dBPItS+X`VnR=p{2RQ8_nZvi6QSsMH2O7!hOQYFYw!*FY@k z(Ka`iWdcZ>kpoic$W0do(*}@K)GE9;PM52mPej?(hh_y*Q&9Kv-nA^Y#$bD(uE?QM zjqGQ!~ejAL7I9SZ )o*R@2d8*^P|Hdd+H3DOot?9-0UxTlwZk%~we}<#~!IW!bz} zf9!-x%mmqeR<^=VeL)s!U~tn&nz!Lh%|tf{e7{8Hg~T1p%w~_InNT6qOY+cAa}VWqp0g`w;Y#;6oDFKKIgi~C)^mR z1rioZB&Yx%;DHG~w38i8bLgK|#t;U~%QOKEInG_!W625ezV_u>;_30!DewJ={B4l8 zG?Qy^oD*%Z1AevJ@+DGqE~t9O@DRG8E&+l&N^fC3!o$|`GsBAf6Q*;P5m8aGndA=L zLm=K3OOi}H=RoUb`Yb)VEZ8+rWo%>F;B>U}$5Z>eH^AkKI!O087K-;x-$0)?&_Qf` zK8o`=Slk)bceAPICYl-Qsm9k>@AS(73XgLAmT6*}g%h|C*0s}(^fPw=$xl1osxNoC zvs91eIqFSIdD;ta?C2JU!Nv+F$)j_eq)L_q;R_Gf$02!zAm^%d^aqewtlEnlD#qe< zqIdYmpNZFRkr*F?lvM3TAOh}bg17_C5oDYee!@$Cj-+UiHWJ|T{Vx#b|DUhOPM@N` Y!kRL@Z$UeNxicmRKL?=lO>qtvX3L%u_%#@@ea&Cr|k|H@~<*?;kIZV!{aYaIl zImDb7nK^FS#>{N{e%94>-{0%HulxJY{rmm?_rlevv}J9Dfh<(_=bgc zyqQs#2V}76^~OjH*iR zKYs_xZnGE;AUaV-gJ;!46LUAjravkw(qMg4tE$9rQ!QVJ9Q${+{uWSf89FV+ zFl-dVW9Cs)Jc@jRNTHPoM!VLcr$sJ4#LhgqWxd6z?+;TCyBao^9do{fqnhkDOPMID zqnofksbZ&^N0`$+T)#CS&*A|Q>tpVi&tsWsqKMu1vve{fw{HMUK;9+*ue0t~Gq;O!MmX#|Q;E)36~FXN5~c()fiH-@9Yq zU|HqYp%_CdR88CMlN3yb&JJk~sn+OGwm=Vea<8Wsm|7-z*G%u%QGHIdvOu*slwObF z^k8@sd3dzid@OP`t9GGLlv@J2*KS+97INwUV_e)UyReUO2D)a-q&(6JiX(^D8 zbwo|4gq;t?VUKgz$9(x$>{N#6wDIO>)k!>h>q}lFbTuP`(_S-B>dqmK&sUQAN-g1L z`DQ3q?RivqZlUeFMnWQGJ!tAdc<;!_JcT=D1bzELqrr~9h(!kU2(hwqDR+@EcLVh3nHYd%m!zXAYz^1wGmI7k#9o^8> z5y@TT)tn~^?62>8!YS1V8cu7;eST_c0XjqoTRN)cGk!LRO-8k#9TF#w zZzL$Cyf*x(M`b6hlTq+koav>GM}wCl2}LzC2cc%@ zygK3I1GTCQvaUxw1?;NIl2eW$mJRx|$@;nYSxHJ}v<96JgR=#73JZBmcjttmSB>Ob z*BfDExB4#j+Squ;U2}bl+Q?>L_>RuzQ9xJ`BPZG(If&Ozk+Xkk-JQB?+9Q)<91bXt&RU%d zhCs&bN#5#L1}G=MnmSl+bd!d1ibhhruOB{cjXM{K2`%NYn?-z$8*D1k= z?{F4$qk<>mjaKV|bC6?=5n&LOK%Yie?>UcyOn>-JVxa>Cc{qN$rcB6pyzVm+F#Dn! zXxkl4hxZ9MIn!h{71>8q?7fW1c1Djz^~6@#ui49L4iyECo)eYd^>o|!^-NrNIGi^J zkHc*~Vzy**)&LXfxr;3M)YqwFT@Rx6Eu4m9ePZqGaF0gA4X-rsXgBF^(X`8yiPW78Rr zYxyU-+{kIWAajEQ1JhW1gD-d3k1${E+1ctRUJ2!|GPoWpEHL8DwG?@J&DU_zRev~F ze`BKC1H+H%r;^i@d%L?W(Y(1*!pgL-E@ml}wEB?{J*2%oV79Pwg)F0H`^p{OMZD?i z+%v_j8V+H!UuIosL+B;PQ#OFsIb9O9E^mEwxA*O3GP!9?lKbTKfag8b~B+vV+%~7aDRqUTQG18eOex70l^GMGuKc&ubPYo0smk zjy32x?|o@ayI_eUp7pr4sj-EJK*Ck)h?h^SQud2;)A5I&$beNI`7Yat>-6o7Tu-hiyFZl- zP1l?}9%CLGJ;#0yOB#Pj#0`#_g0`I#IGdE__H7RZJS}#$U-W`{F!*U4iRtF5jF7uV zJ7cSP@Vsc`t$ii38C%BcW!3Gi4m%a${a-5PU?S@kmgr>_70G!UAuyv{{8+=TIt|NT z5^hW!@7l3_-N3r7tfQ@0@V8is$~B$`4*HJ1sG6@BS`C2s=c2@Rz&oy_B<1Oehek}7 z7#;Kq?rg1!82T8xVZ5G0WiT2EDvi4-FORw#CO=h1h%GW=qI797aIeR;gFMNV7X)Tw5DUB;qYPZM>Fqh!*6wx-pe*$vp_&%!#-+=TUpMJtP{KA1ZQlb(SoQ zKfJ;1GJjJ^P~IT7r3?Q^$8~HkpKc#PDMCpDky16Jbp#jh(^J{oc(*-Wo5&CGy>X~* z?DTF-f6~y$`JIw0@8m+w05N?PP@fZZ?Ch6!L{5#K0&RP}V(?xB^R=78{17jh;5IOo zH#K_qTMXURkacK^vZW(6A?7^u!bSOk4Q(x5ZtNCJ@et@o$gSke1RF<3^>zvrNtFCh!>`&CGRl+A|rdKv_+bt3qZ?Y%a6v92QxC$b} z8^EQ=7ZVHaRzR@8Ja+YDzy4H0KicZtSnRI4!syFHiDAv|J}bT6{l{=65{a$jo0Sf#M58e@u_Wo4!8E60`RM&Q3`>H305@ij<@lC`4e%l zW%K;{FBr5ah$$=hI)PC!p`AS;`E12uOil@Au|=n$2;1rx*OUypCw2g6K0qp2E^9!b z?5E*EAW+;95g8Cj@6&(q1^4n{TGUR{tp>>kb^2Y^mG;q3_Tbm*?Koho2YrsbYJozy z+{tV$c^tIbJ3G=~cGFbKP6pJi_kQh)ic{u{lDevO!5z+AX)M~Y$6R|}3Q-O1RC?>2 zk^ybd1*JR)7h+zj(d#v}r`9pdgXnF_vu(#0Dc!F`t#HTAAj3jI$ApDMr}F&ozeNR$ zpIXqXyweFU{xU_WBLy?3S_o6>Jnd zale@_7{^SQotyKo91YG-Jq7oLqt$@2m{@(>qc17%wwlwcnH+RB*464y_E5O{bxlyW zu`(MEJ;%|;21Rn+!;j*!NDXyFlcnpIwYBo??{7PL z{i#E>Z%)H1kknS6IbZU0$-dQT4UPw-hV!7@X}}IeIXO8Yp_iuCUIs7jgYPWot*(G> zh>OT{T1*Q*^7r?z!m&n&PVbl8IGP!u@_6u4x#FV6NnMO*TRj}z*i+5b_^ef3M_ zSoWFFZ!NJbB6$_V=+GvF0xB;Nu|P#01hg8-D3p$M#-UcWzHiZmv|B<~fj&c(mH`L) zLe|5DTU-2$73P#1^Gq6BO0X*36 zA#b?_S_AY>umkmcE)7L7xU9Na&^JP%#!J-J(D^z>2cQBsLe_Z!5k8TYJB2PiCi+n0 z!eYW%9P#2T0V{ED@|DXPZj#`m4?wHM{RjyWF&E$XDPSN@biiW|>49_|YZSpdm9=HB zVHGxPF`pL1+rTV_u*HU{g5kzH}ldbEmo5y%VJN? z9AK@?VrLCfyt>{=)P0K4S zRr7X=$L-qzg871L%d^`vr+_g>eO&PSlT)uA4A;RK%xe})$I8pfmhnLI;sdmvMwckS znaPa!BHAaeJ@+;D{LAr*AusJpAc<<4qh(PPF*(i8WM2rfy0o-MZ0E_ce~94G`RX)P z&^9|%dZk0p-d!GxL{V;6latB#=Br)cq0ewY*3jLH;pM&74s1roSwT)FN^ers9S#Tz z;^0dvjhA$f`=KkYHj5ya6E^E_*nQS0Sl`|fY@1~m<71v76{ zdJY^x+9k)kr9vb?pjY5*=i-UPkV|>yi-<(8s}6g>OW{`M;&ViWbbx8sr31FxE$@CmYh$bHWKk4{^u;(U9ekW>L(>(-V|~~{ zupl;s(;{2YD^cB;iZUvKc3;Z8D&>5#S$eeUoW*+3uks?+k0vneOgxllj7lv}$3wq2 zn7B<|!JYGbA~HVNTSuTi*WmYWpww7fsoSQ=N2%5Mm!Rips;q^`bbMi-9|&|PGW_}O z<{@!=8)p&O82iObwyYlys2hV`5pnT5TUo9e1QW0%OEP zuowyp0QNW`se(;-ot49|Tbbh~Ru?>+H&aWiCZBZE5h#nhScl@1qyE(`fzb{8tCe8q z<^aMh1O$3<_Wh?Q%Py7rP}k;cGY#A0)T8`WBy)v{$u6E)jUpqwzRciXgrpoY|6Vxj z#G6JE6W4$aMFYaJTa0QO=_ICnhlB12AnMe9#XaYJF=3E$;ryz=QI2elDS5in@_Wsz zQxszBwog!Z@A&Jn5Dt$Z!wjn$0OE(e%1VsiobR7$hY zZAK3+vl*z#k&QOkph#FJb^mbZn03)?b^6_@oHWji+dQ=uR#6*Go%XL5ldu_!Ne<_5 zvqq>6J;d5PXRPDvL5BWqARfBhny-R(&1R!lDdjp52c!xNXi~!$Wh-iBQL@7;MVYlO z*Yh}CF*~1YgAx{RdLJzpOPuSMMY{wTb+m7%ZnyE)j1RG#>B)bO_WCU*V5@Kox^)e4oY9s3ZtrP@TP2cAh|@&x17|GL z%GGwxvB=ZhjQ=`t@R6C$i8E#*Cbsz-<*|Vo@T_AZ3`xcm6`{3t&EFeZx+2 zM~n5Aj$8(o5is*~CvK@E&-KuV5_~ z9!YTz0uwkIYVLHtQjtq9!uhe$`%jyltku>}iOTu)=k{f6-!1os&*iOW;$m;f;}?sMoJC1T zuhFcG`QiP~Gu`XWItanJR+%SFhLqULwVv5;A-Ps@%uyA{FDj7%YX{=Oc&imrd!$)) zE=Ek;(ud3ttd7H0IOYijwUidCOj~C*+T?oz64HP5^3bD*Yv_>8`lCo+h^@oHRoFr4 zqNx&n*@9q#Xz!n2{WLAfd43^73c_9vv2uK$kgz}EV4A6A_5c z1=f1$mM_AqQyG}ul{1s8T&rsCi#ih;q?^u&+(5n-G%zGBzQs^tsMQs=sE6_Zk2FZ> z4%F@fVh6-s_wzJ+g+1>3!|ssXcJ+BP#f+iVQ8o6hh_B_{ZZG#MSDW6nU${=-PB&@# zp14qdfF7O0;Zp;4^5p6aa1^0ew4WMmVx66q&w;CJaN9eLbyHBA-`|_R;bFg zLiok< zV|fEv2(;}k5TD)o#oJ5)i02Y^lXOp6ghr9IX-ND0K2qP?fLeQ)h@FSCRX3TLcL#WZ zgy02N-X&is&YVa~!mL*J@NaI3XL@@~5B08Cw%Ri?6Euu~-PZ-c>#{O_>>=TUVG58I zW09w62h@;8ru*1K_CA&Qi#c)SlLkso&w#f2m5bXsY~$5KDQUC*1EEgTxl~Q5)O|Zif&!fbFJ(F~-?yueyRzmrFVG z(0s@QetWEj^j>z+F80yAHlrl+{!v@DyWu#m!n*LM)MuB+TY~XIf(_nJ1y}3Ne1hUJ zlt9jzdvvX`iL`_4RFBi;4TVZGBSnKKee2Y`hUIsIz8C-bKYzfkA2|0T_dxg502Lo= zsLHGZ*)2_E)@e`L%;R_$DkEAM$&}b@zn>6#$vgetwEIf$At<&9%qK{idulxq%~#_# z{T@!>JN@fZqZOdhr;q#us(banUXks;X+;$t?^#1 zal5@fzc}pGkp?kK*Xk6&tsbmXZgcc~vEbp~{%NOg>u#J#UzN|dVW91G^zg_}J#-~p zPCL-^5=HEo0E#iqj}2Esq}bcruacEaldE)=`m^3J z*6;Z3zzJS=^>Fd&*wp4-FLQyxD*|Z03!H^K`irk%+KO=jfP=?5t%yT z^#-n%Vn2x-@nDy^oW-zT|D_Cm+bA=KQ4-Y@Vya8yXy=Us~8v_2?xIAv;X&$^u zeZA(CPwBROrFNoy!_|HxkGJd{{`&p_H;+G|U62C#O_8>>D?mz9f{pYcC-#J5lec;r)(rx{(TH5~+6)*I#oigjRnf-Q~JUhQ#KYZ@d zZ+LtDmCcRl`#S}Z@tZVfeglL3ovLNczl!Sw{%*vK-=b4;N_z0BOQeGK?*jb{!i`Lk z+lPzK#HKg@x?aai=E3g)`%7>{#kT&8mX6=!;%7wuExw?7{oDFKPO!g&=SPF{cj^kD z@PEt61K`(n0(u^pi-Ri{`y6W5iJ$mY-Lb0pl=uGt%6|@#^D{yo0GTEzXxJyxp&<;H zdNSaxHh}JUhF}z*YnD0*jN?ZhZj>DEf9IO|6CD41_P+cgYHfu;)DB%wz5R*kW^u{X zG`gV)CDMv-P5vwd{*0Nx;BQL% zZ&>)fu>Bhr{)UBZ`~DXcq?@z)=S8r*R$$os7~j6~3Qyoi`8jzjrNTb5S?74o-?Y^D z^;d`1La_@gQaY+z1XX!XAL7ku$f(EhKbG1|>$_}(2LFq@lMiJT_)jpogue}IxZz=&%82Ecj{af>-RJp=s zILq9AF-bsdpN>+c6+#>8x*R5>ZiACG;}ioTlw_*_8CDDsn$Liz%ru~8+9^2eqkm|% zPJ2g0@TUnkRVo^09D|}&>*(uHNZ+TVSH0esVYWaipSV8%mClG=qEzoo!DGQumFd!L|V-|HCAd)W&Uy0rTtu)~~| zp5A!#Fs#@xR$)F_-C+_8cB=Ujv*WmlI=)l7mI4Qv`cttbCDflwA9Bi;dD ztR%1njFwK)g}m6y+CxPYd z?aGl!UB*j-)o0-yKGQj<8h{NV;1mS)m}#*Sa++}x@#;@CRw4ps``9{mJ`+$Cw%O)> zs}iE#r^M0XHiX5fyMFy8i>0)+a;732!2)}ts|D z9&Q-qY#wNw%Cr#hV&;MEFfaP5cLO(2XDE^XdSXGWSf$ZS z%xkZ%T*2P&l&b*v-~?K=2*u@{uxP5jIHaKy

IMePK5AvF8>4E^w=N@JgL5 z}F?m3FvSjT2W79E%?of^$-#N*m@Do~%{b{|6tJ#Y~ z?~yBZVUQ#o1L!vjoImvZH1l2xZmybw=1GPF?4Z|zI|q11Ai!M@!=LGyVqIERD9AVy zxq@Vm9`_VcnQ7S>lr|JNbepRzf&#;eWWO_i7#FyD@qdh$^`AZJ-sdSQwJ#C$$^u9y zrmBPII|Im>N1J>&^OYFH%$q#s)pxqzpIA&8<661<-`(ii!vcsW9RTy;OX$J_MStaV z{{(>W-~NjoSwLV6p@i19|SR0TtSpf4W^eUqPM~T;1R>^Puw;1J~tPV-m=ICm?Xc zg8d1oyX5CK!^%*$F)KqNw6#4AQW}DI30xK+*6s`%?h;ct)jtwETS$gqbhur9xmh8< zqr<#GS#?9KWT;~sFp6De0>j)YaOkwD7w{%Oqn!as4z+=|*%hFY*l?j&uqQHVD#s}p zWWB{MM5$Lj`#Q0!Yb@6SLwzKn=!}-d6(-3ItP22G>NP;d6kdHPU|5II`qe9k$4#x} zRSfdde4~I%CyyMe?qr}x@;5}aB`VkY5p;+!9*@;PnQ9!H??>|)uzZ_852@61l)e$I;*Gz z0#{)C;q~?Db9S6@*@h{Ri13Dbyhd%6qXq?xF9WP~yynEcr_p?j5NPFGS(Sf40J{kR z?FA9&*1sC(~z7LTt6D&9WbxHpo{L3EyHT`f(I(u+~3+Nw$KJ+WD zUx)xF<+T}q1FjMH<=bEbTNIHY&2(Sg-XZUXd9Y$Avx`c{gM%b=m=~~q#hJ`SJ#h&% zK>DRF5@9Sv@qY8PsfLQ(8!8hS`C&keCgll}R{AQ3O9{HO$VaN!x^B!1374;rf3pp< zI3*5Ttagsinxw(B#zDa<0E88H{RceE|197CNdo2YL;9HhykP%>fJ7i;1M{;*`Yz%B1z2@C+5i9m literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/huggingface/img/qwen_tps.png b/end2end/liger_examples/huggingface/img/qwen_tps.png new file mode 100644 index 0000000000000000000000000000000000000000..cbc86c8f41fddc118861541158645b632f2825ae GIT binary patch literal 15408 zcmeHuc{tR2-~UucbyMk_LK{wt#9*SNY*R_Hi|l5GP+5kuZ!?Od2pNW~Lz3+KHq1

IMePK5AvF8>4E^w=N@JgL5 z}F?m3FvSjT2W79E%?of^$-#N*m@Do~%{b{|6tJ#Y~ z?~yBZVUQ#o1L!vjoImvZH1l2xZmybw=1GPF?4Z|zI|q11Ai!M@!=LGyVqIERD9AVy zxq@Vm9`_VcnQ7S>lr|JNbepRzf&#;eWWO_i7#FyD@qdh$^`AZJ-sdSQwJ#C$$^u9y zrmBPII|Im>N1J>&^OYFH%$q#s)pxqzpIA&8<661<-`(ii!vcsW9RTy;OX$J_MStaV z{{(>W-~NjoSwLV6p@i19|SR0TtSpf4W^eUqPM~T;1R>^Puw;1J~tPV-m=ICm?Xc zg8d1oyX5CK!^%*$F)KqNw6#4AQW}DI30xK+*6s`%?h;ct)jtwETS$gqbhur9xmh8< zqr<#GS#?9KWT;~sFp6De0>j)YaOkwD7w{%Oqn!as4z+=|*%hFY*l?j&uqQHVD#s}p zWWB{MM5$Lj`#Q0!Yb@6SLwzKn=!}-d6(-3ItP22G>NP;d6kdHPU|5II`qe9k$4#x} zRSfdde4~I%CyyMe?qr}x@;5}aB`VkY5p;+!9*@;PnQ9!H??>|)uzZ_852@61l)e$I;*Gz z0#{)C;q~?Db9S6@*@h{Ri13Dbyhd%6qXq?xF9WP~yynEcr_p?j5NPFGS(Sf40J{kR z?FA9&*1sC(~z7LTt6D&9WbxHpo{L3EyHT`f(I(u+~3+Nw$KJ+WD zUx)xF<+T}q1FjMH<=bEbTNIHY&2(Sg-XZUXd9Y$Avx`c{gM%b=m=~~q#hJ`SJ#h&% zK>DRF5@9Sv@qY8PsfLQ(8!8hS`C&keCgll}R{AQ3O9{HO$VaN!x^B!1374;rf3pp< zI3*5Ttagsinxw(B#zDa<0E88H{RceE|197CNdo2YL;9HhykP%>fJ7i;1M{;*`Yz%B1z2@C+5i9m diff --git a/end2end/liger_examples/huggingface/img/qwen_tps.png b/end2end/liger_examples/huggingface/img/qwen_tps.png deleted file mode 100644 index cbc86c8f41fddc118861541158645b632f2825ae..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15408 zcmeHuc{tR2-~UucbyMk_LK{wt#9*SNY*R_Hi|l5GP+5kuZ!?Od2pNW~Lz3+KHq1

+Ah`Khx7u+sl1~ z8w3LFRljjf9|YQN0s?Jw-Mtg|WQ68f0s<*8)vsN-sw?v*DkL|hjEcWuA$o<;Y*R;Ey-kg?fe|!Ax zsrI)&UfJgSlskm`=~IbVF2;ve8k;(A>R((;P3mWDdq3ybMntqNc@JJ9oqIpSVVo8A zpJ@K53miqyaARicNHd4I`9Yw|*qa5wFImq6xPY&04@!VQ55?0gw}C*n4s-E5JrqiYgh}vg*x&&2;`S$ zY53d!K`biHR<=FadsH4tO{y=-S~#S1MK`$hrL;<`{Io=Wp>?p}9N%;Pfo;m2M zYgNTL##yb0HqP`HN7Z-L_dwGIt*7YC4*7%&aqM~2yzgQabwkSg`E8$p`JyJNsHIi~ zNL+9egk%vTVcKCQyfWaBFA|XnzFv2GUYI?IZO!EfZG74VeNo(|Y@S!W*f`b(p7hey{+uN#X0HWmrCUTZ z+_rogwH)+Od01O+sLPgmUxDGf+Tx7vv?bv<%NbScinG?y15~rZ+p0yln6%rB2O+`^ zx@e-j$5M-&C?yX+mG%0qm?X@{svM;l)>;_8%c?5oW|i|on`cb`7wm%yXMF-Hz%cM% z>%?JZ%{l`5*z=99y|w8gt#cVPA4(ZAXk)u1O`hmN(@^*UbpQ1o?jBVESajlv{z})o zi?0rs^=~W>q~h`TS89_NY3bloZO~1>qT62vUE4&;-K+96exM~s3=!Sv9ksyA(5;o1 zn$MAW(uO>ZNxNWz-XqUM@rrFDo{{~4)IEHweu7DMZp1Q z;x=aPlG{XfLlH1Zvnmgyesxc-d2G%hI-_3Lzq-KLODoBDC?hrXxap4_NPNJ-$*feT zPS2}Tqja1=a#h}0zRufkDW&&!7BZ>WiM%?ka(W0Z&rJ5hh12tcUC7V=`$Lr2GsWbW ztp`Bw^5ex!AcSRos|4b1`UNFE8)m0~mnNdxy_wjeRIRCH{3^hut=@?@`(_>Oiqc}k zNf=|Od;W9HE@25rgU{Ymp=is^=Ps1lS&ikclsS@J=V^soj6#oziEOZ3>J5trb$m&U zSZ=P2RoHV4T&z1PW-To?C zA;u69X$Sla;;57ZMu5eWv1maYHP0}Cp z@xyM4@f#&92PyB4Ft9jVn{{PjG}gDd!2ZC};0)tFCGUrziA;e9y9(Pn0f9sZ>w_QB zgt7dl@o?eRmtI9(=?~5*|Mu-uriHbCo0dh_{}6- zP<+VWIN7&Pj*<4W8{BPk#PAB}AvE)h$>$+7<4EfLM=lGp()gm=h%dEbX(RQ$ooYT5 zcto22lcC;WsdA;kGG{jLJRHpFd)7B$=WP7baQhqFF(<5V4vfVbsQ(y|_K{-Nv5YSP z^k$?kl!ww-7-4GXE4{&@EBmxyJ0?=*FLzHE*z>g&F^vJeSs6;jJ1-<~*FxJ3rjAG? z2?&lJ0UVJe9~b!GLBeo)SJj{pYPr3ZxR!Trh{2|ADp$TyqpmPVg*PjjMyEqFhI{y) z3&6dO_ZthF>k9OvW@lX}?~y1i*yr|%%=sfc;A?B{``L=`#m=Av%%AIdxdnX?XPQ(k zE|s?F;;PVjQBGj=-Na#0#W!dw-u|_KS6hM{q(4tMaDQLDM7h$sU&$&RmmR_S@}5_e zvMl&iBY#&XWo_;{c${Mo$Z_@JZj@Xmx*$7G&n~hI8u7CtbmI&!pjv=LeNdr91IZbK zV=$N)Jo6)OUZ?Gf3*Pjv2@kMp+4iPg&d(=Dw4us~J&A z8UP#!ZMv2RGE;=(U(wIl`@a6(z06&pRcuH=`f`vK1m1&5$>??Cx&aMsgul$Sy<&W1 znM)^Mt)t_oJm12Ie6Z}AsVaWtpmMi@pHAek+!=SOJ|Ipd?R5V<^y5HfT^Ow(Bl|Wk zgSda1rvlx!MlgQmA3m4CEX4cf^lcsY%9#smbJdq2;=R0z-3GR6FMamLqtDpQKtCJU zS8=KWN}5vK&yqlJVpbLd8$_2I`YH~cl!jwcf4s+fj$A1j?!WgI^(Lw9Uu_#WEJs{s7UGtgT6!r7(PipvL~qF zx-JhN>7xk+nbuNIRb_t|%wv;1hOfH8t zDl?9g70;d($3rv}Xz{06y*x)=y<%^_EUQ=W@QAJEt@K+>Yb(>V_rjclp}|~F5_?F5 zl*d(e&9HG;uL&v)^4-P7^Y&)hWkZe9^*K*c6vUs^@P> zX@~HOCaEfkO4N{hFZt3$Vw9f}dVTe()kkD6+&B09`SYjJr)P@WoczzGSv@P!_|EqKC67stIMk@~E zw-E&KyK<*3SrW%;zKfx3X$Fnge>HGsu=1R#bo*vjgx}c7AVV;$McflPFX(J8O0Iv$Fq^oPShxMYiJw@K(mi zV_nzCw|fX0{kW&6N5^`t&qrYl_!wcO7xB0jO;gHw8+*tj*%g4OOB7^wix1`_sGH>{? zixuB|d4}Votms2nbhy!<6-#>ACrjG;!zqDVU;;-i|2TfJ5gbMFnQ|De_72)uIg*Ol zm@@9@>w9@H#2Gbh9#nXn#{J8dHjk~Hf(^CT%TJ~Ra$EbbfyzVEC}27M%?P&Go_{xR z8<%VEz3Ou`q@mSy}#sRFejR-AQS^IQo37)|qgb z{rG^+3u6STd#_0>uT537K0?j-BCt z`cBD?U|lmOqPKPNVA?GVy%xg4>sJ?)B$MW!Vzw&avE_9TF?P51fw-bxU z4OB=%HoaAQ9f$!FJ+!f|GWH$U*>X)jiSCb>STLu+favSXny&cx2*3?4^MZ|^-14-Z z8do#dgx`Ztq0K@{6zO&!T!{X6HH+kJU7Ko+OpC>0a}v4ooti~ddB1`VlUC`CdU{Sd zL}>_9DJpWa;T zHueuyANW!5>(iVye-eT{(%I>?W@=>ia(P;RW@A~<-@{O&crxA5rDp9JQWdHBtgWOd zWB0s8M|@&@7Zj0S*I^L;)-nHJPD_GZ)0JIjSy0&xEyGWSeMn$!JgtbYK{jV zJK;Lfihc9ubeyb1%rVzy=^*JLG*E-{%???0yjGX-WjJQ$=f~Q0z9Ike>_FNFU=bgc zUjCBceHTB0;?CXGAqN=d6c*lOP@GA%vxm>!X<$v@5G886V5wQz*_uQ+!Kd`OkdnOp z9>5ai+^7K_?*Y^SvuPx5;?SFiJsE!S)qUUf)djNMd#m?P$n9#vk`6sNV$b&;mboSq z$MC#d0>BQRY65@TnJIaX_m;ScfW`es%?Wb(_u21u@J6$iCPZ!R+Fz;4Sk*qR?R&Uw z2RCA3$(v3k`F9-xa3kkQBG>GrIqn{y@EUu5uC*Y6$E7lHkDz?Xec#nDpF0KyI2R0H z=^cjIFQqjhuMv1RvEz^TU%e4~Pxo!iBCsPCY~o}8rOQB9yHW2FKBRP$!xnIgIDGE< zR{iRi%M_NizxC@*XNCf@Qf}3BMA#jYbFy%GRFyt|w#y*UERSEpQrx65yb90ICs zEEgP2{2C5A^3~ptmlb&J+4^zW+hNMZr8B^Y)tGqNd!)NH<8n-HOJip5dGc!L5!->> zmLfYf;#mgU_AqM@8%FM{Bss<8?iIB5gdk@ZYol8XiHI*tY)_9$(s-TRu+NJNJVtkD z%hT~TwGpOVMwH#d8_wsbT&IzYuA}@-wG{yuE zc|u@th{M~R?*z83M-09^nEWQmyYe1fQ%0!S`(DQS;^(9w%9V6BHcjLU57Ys(ZVuCG z!Y1dLuHL@jeQedTDdg);pkW!BfOZmhPh;6n%Q$1VoPD`NT}I*P4z#Zd3Le8h%kIm{ z@tcj+4x$Kf@>J)%#SJP1Z{ZP_1gMk~<{&o@c`Mcr7IT)}g6jFjhux!dC=sggct+*nq& z!5PI|?{6O@>KjsD##-N%YhcdXM=n5}w33;cN9A<{UwQm-la((#hgsu$TY@MG=Vl=< zQYX(y>f>iaf!ofiQdSNQcV-uNbcin|)a5SHHIGKIjqwOEIp{!qjwP4bn`NR;Qf!Q} zAa2lT9PfXyYrDCT;?4(v+I-uy3bV6$EtK^QSGZ<^1M1)eaXgpL;wbZqU3T zgY=B+^Pn0J&s5|mt}aLrb%gQ3}3{RV}Q%AuH%=Gyk_PUAnUWY zr*5`<#Q~Pe4Ig?LbpV1C?hG9jHW_9q-acb8G?X9B2IA@FU{~csB(hmFNcdY8q7`V6PwKS?D>n3S(t{sEoF5Mdn zv>A$g$cyA6_E1#Z*uhlzOy2zUE`!I|x$@hn2}7xkLrS`cqf(lm=Qa0jCO@xzpgTx%_!X!y_h z0+rLFQvSK451NK_AwJLr%!gWBs;w3@U2}(Zbminfr{&*^uD*KvYcOS z2{d{9f6$M5OG4=5CvjD;ZvG?1DH62aN(;Ii;%(Md!(?WA?(~T9KuWng0i$`T?9eJ*oK! zD*WBmnxc|3sMOqRG;KNal{-2*6%{v={gp(u6_#AutNB^2$d;f5gMpBtu!xeziG0Us zQMlMfA4sqHH?6cE*nyxIUfI>(Qsl>SIef|G^`0Z=Z+r+B1cwShhO@1|=<2p78Qv=B zK{_D}^+Vx}r?U+n<;l=%s2DpEAy*8SXx3#|;KHTJWJPEmu~l>JH3*GK-q&R)5gI_F zl4IEt#>dyzKL;i$L|jSC63D@VjT=7)3rlURxuYi4qfV0n1U#Kcgul^FIDJ4=XAs!Y z8E|YLV0}O9=C?L7%6>XO8Xj}Cro*9_=&hp08i=#oYywA%DA^t|deg|4lOT-LHYyNr zpRPE~DY-dS<|%pmpv2}3&(r#MyA+%On}nHv6q5h+aEA?$t&8x)3o zjY8mx`KsEwhA}F@`kdrHtZ%5E86a!hA#Zb-aaDD@I6s_7?80w zDdW7V@6}Q_HGuFZCHCv>A`ibczLWfdWDP~neoFHE5r#`iOr zZ~iXrDEuDS?qn3(Eu#~y&?}lQMWX4dXErG>v|-Fqkp!NPqkmBLgE{Bz?Z7g;bH){M zIU15{mh7-`Sqyc(KDZ_gq~#9(lO?)&xQt77tpM|ZTfpdDZsd}-~1;G1SHMt&`GA{}_sqBgXV3q;qMB#0tuV^~X% zPQ>MUygpZq;DjYmbQOpq)Kz`T>Lz$_r%&kbHxCZL>f`03y)E_FvU(Bg?)n}fwz#!h zGBL1=S-Bvjm}9;h6zjVZtj@}M-Gpv&IJNZgEZN!n4C`ERld?+9RqX=E&wtkjy@HOn z?$+j)e4NcAy$f>HWlQ*ygl{m&*0XS=NCw;myqS?Znb?I^7)P^=$P!+-F;7K`ca`BC z(%>56bg#FqkdlcDm?+g>&mneR{CMcR5<8hd@5rS(}wr)slKt~aFeOo z*q=5pKsZE<>S;mR(uYncP&QNNi`ewOvigC|LD2}T-^K}rJ_t6cickb(n( z;%#{EMLZJrnEQXaoEV1chO^;c53>8{_RvPf4ld4#xTp-^;#x z)(c!G{vm6yD*hmU@ZV;(%}RfH-sBajuJF4$F4|BhF|^LOLW{{=`S{$0y2->aGG%|~SHIdnGVY#DlEO=`4ek=A58 z)(|Ok@8i#h6+Chz;MEuHbMn7+8J!6JoZE6#zv4uIClEy0s^>?;74q&y?29_Bjp;AG ztBFDpv=T08pVmn{B-_*52UIPO{^{v6jJbON#scl53NLnie7v^oACle+p;Tgsk4-+%R1<=qZj&N@ro)U4!Qq}I!`sb`J+a*dR6v78OI zTG)y#kXCE`v&PUD0^BLcaxz&I=P2iiX`FqF z>?%7v`R4#?f{fGP#r`5&O)Ez-MnvZJFS`$4yl=UfcV%NH`(+{V!?$!6xJ3c;TRH$l zlM~+(B;a|Ue(Raen26m5g!aQ6u^M zHBkVFrq7dv^{Wbj0Q}EI3lPi1WB%%FpLJj}l?1;dFb6J8<3a}2f3tQLXlym$-KR16 zy?HQj+V!x3e<7@Wm&RUwC?K*)&OiW!;F5j)!!0={1u1ffHbb@Pl3ErY*S;iynRZ`dO+6;h54xr z;5pXzVMH`-etG(%%>7UA#$bVK{l?PY_vlvhMgQ}wz5jilH07~h_85FE+(yVd!bZp^ z!zLDW-DcwGgHe{&q5=lRi4NbOVw_EqpVQ!kcl%Vg(t@*iHZ1FW>%3Vx&=dMYK>(u} zp$&?3*F(n=jGmP^MGuDFy}?-EEhpI9f+I*PuVTGdVJG?plZ2V+=R}q=$`@V*2RQl1 ziia&#^&o*B%^ymX$jsR0Kz1o1g)mqsiHn-4RXIgsF|qaUvuX`SPchfrRoNv-|FM*^ zS=^W^@2vQSl;M#H&s+-7Z~WbX1-5Phlg$Cl{uT_3Q&%RCZOT{wJ`|4L2*beE7HYkS zPI)%5$4*xAQ9r5jKGPs+Vn488)5?De1u4a2b?Uk`hbpm~Q5HM~c8b-Louo2o6XJlfd}FvnJ?N~GiH=T zVQgQ0VTOJ1fLcA!%99S0B%|E z4WAg!4v@8J*iW{RA@@p3O3HYBu`E`jT74tYJ;vm6_(L^7x>w@-Efi{NLBG}ec1WZ) zv4>9r)-FCVm7O00(E8p3T#DVtVXu@@%?1T>{yD*y@delm%M!pQQ(c)duH#yE_wK!* zcV;w=k)vj%fbWzVQ8i{oHqV|PlD@ig-?5j~X?jqBt#Kc@O32o{Pw5k&ar@VKAi{O( zi~C(S)yR}#M%fq%Y?O26xuivf9>D)Q+wVz5RXyM4I|9)CqtP4ko81{CgfP3W-iEZn zi1J^{@W&Wt8@V6Z<)L~Z0o$nyRbrt!;JQ4s~EE26f@ zlw$hdV$-NHfAFBzLpcjl5Jxp(R0%*@(Tz_?pv_gy$M|z5{O8W^n+e&PLTOcIJK8xq zMgue&B{5T@OsdLjp5oB>0_*y48U@pUg=X+t5;iUE=bH z{w*!BMF zY5@Ds7vt|VJ?uMgTX)cROmN;S*y4T-myBay!PtBr!1_z?^s4;z81PaQ+rzuJY8dOM zH{#YE2u4xmXot_3S`GnjD zC_X`jbH5(Yz807x5})^O^PRb4TILv+_~64~k8=f0zpkYw3KuOkXDl_AB2J)d&I^@oUnit1aPDPPVAzb(J zGXN+q4MQSD$1s3VquC&7w!#gv2ILMu;rixm1F**_7E}>-eUs_G$*g^e)_Py@V3b=x zXh*jVqKiXAa70+JF3;R4jTp4ej)E$nnHp2gHr<@iGoFC5``ZR!KQWm5>9sSjdMDi> z!fPLcF`m1Hd}j(5fiWV!5Bro?EjbWg->?l7H!e`t>&vGxlHc4B#$)~!2#T(`m9vcr z2IZ4D_#qK1|F4hsKVZD=_aQp{w9?XUr|l2nS8BHF833GU;|2it7;88g5iOeRRXbiWF4vIQYR@c-Lj>FVy`YdyNMVwsac+ zfmQiL`3!t+Gj0mA8q~&Kg-U5?bSF3y#SrX)-mz@NM%K_?kVXhQVJFDvspRD?5_GCc zFuxO&(PzEb9)Z3E(9EEh05q8P6Y}GYH{{iHYB$7^`oGwcvru<8@`F>3DP8NdQKMJm;etH-+V0ZUKhd zl)pUUbsLyCN?pBPB+p2ouAde&O}{55zp>m&!%yc$@qvx{K1-?UwQSA2F$wb`2>)qw zDh9}KMcNE_hJdj`JI2lzcVL@OKo8+I#*fd(TDUwLl1vpS6%a@o$uIRn4ptQ2SNMo> zGt61_5KFj##B9;ew}^bA+4^QP$$pFE+6n?LF9$VcfWufw#SWsW>|xP3tyAj%`j z>kJPuvxa4}f-UYh#1TxyC3D!on4OfnyL%847`D>Qz`PZh*)gdgzS*)Jq}0GKq=aS0 z`mrv_lq`L7uJW22>;l!*kv>QTY@Nw>W*aD((-Tdq2$!b+Ec zEl@iKQKnJ?_l;Pee5jnWOqBqA)C>vLtk^jngd)E7vqJl-;Z@D2O=lp#v%T@UZrT0w@7J6 V*7;Z7!oZZ&@|@~&D2{2zCVi0c3V diff --git a/end2end/liger_examples/huggingface/launch_on_modal.py b/end2end/liger_examples/huggingface/launch_on_modal.py deleted file mode 100644 index 1171ea4..0000000 --- a/end2end/liger_examples/huggingface/launch_on_modal.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -launch_on_modal.py - -This tool is designed to launch scripts using Modal. - -It sets up the necessary environment, including GPU resources and python dependencies, -and executes the specified training script remotely. - -### Setup and Usage -```bash -pip install modal -modal setup # authenticate with Modal -export HF_TOKEN="your_huggingface_token" # if using a gated model such as llama3 -modal run launch_on_modal.py --script "run_qwen2_vl.sh" -``` - -### Caveats -This tool is intended as an easy on-ramp to using Liger-Kernel for fine-tuning LLMs and -VLMs - it is a reproducible way to run benchmarks and example scripts. However, it is not -the best way to develop a model on Modal, as it re-downloads the model and dataset each -time it is run. For iterative development, consider using `modal.Volume` to cache the -model and dataset between runs. -""" - -import os - -import modal - -from modal import gpu - -TWO_HOURS = 2 * 60 * 60 -SIXTEEN_GB = 16 * 1024 - -app = modal.App("liger-example") - -image = modal.Image.debian_slim().pip_install_from_requirements("requirements.txt").copy_local_dir(".", "/root") - -if "HF_TOKEN" not in os.environ: - print("HF_TOKEN not found in environment variables, using an empty token.") -hf_token_secret = modal.Secret.from_dict({"HF_TOKEN": os.environ.get("HF_TOKEN", "")}) - - -@app.function( - gpu=gpu.A100(count=4, size="80GB"), - image=image, - timeout=TWO_HOURS, - memory=SIXTEEN_GB, - secrets=[hf_token_secret], -) -def launch_script(script: str): - import subprocess - - script_path = f"/root/{script}" - os.chmod(script_path, 0o755) # make script executable - - print(f"Running script: {script_path}") - subprocess.run([script_path], check=True, cwd="/root", env=os.environ.copy()) - - -@app.local_entrypoint() -def main(script: str): - """ - Launch a script remotely on modal. - ```bash - export HF_TOKEN="your_huggingface_token" # if using a gated model such as llama3 - modal run --detach launch_on_modal.py --script "run_qwen2_vl.sh" - ``` - """ - launch_script.remote(script=script) diff --git a/end2end/liger_examples/huggingface/nsys_mm_timing.sh b/end2end/liger_examples/huggingface/nsys_mm_timing.sh deleted file mode 100755 index a68f42e..0000000 --- a/end2end/liger_examples/huggingface/nsys_mm_timing.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Tell bash's built-in time to output only the real time in seconds -export TIMEFORMAT='%R' - -echo "Running 5 trials for nsys mm; printing elapsed (real) time for each:" -for i in {1..5}; do - echo "Trial nsys $i:" - elapsed=$( { time nsys profile --trace=cuda --sample=none --cpuctxsw=none python training_multimodal.py > /dev/null; } 2>&1 | tail -n 1 ) - echo "Trial $i: ${elapsed}s" -done diff --git a/end2end/liger_examples/huggingface/proton_timing.sh b/end2end/liger_examples/huggingface/proton_timing.sh deleted file mode 100755 index c7ce1d6..0000000 --- a/end2end/liger_examples/huggingface/proton_timing.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Tell bash’s built-in time to output only the real time in seconds -export TIMEFORMAT='%R' - -echo "Running 5 trials; printing elapsed (real) time for each:" -for i in {1..5}; do - echo "Trial proton $i:" - # Run the profiling command, suppress its stdout, capture the time output - elapsed=$( { time proton training_multimodal.py; } 2>&1 ) - echo "Trial $i: ${elapsed}s" -done - diff --git a/end2end/liger_examples/huggingface/pt_timing.sh b/end2end/liger_examples/huggingface/pt_timing.sh deleted file mode 100755 index 0bde599..0000000 --- a/end2end/liger_examples/huggingface/pt_timing.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Tell bash’s built-in time to output only the real time in seconds -export TIMEFORMAT='%R' - -echo "Running 5 trials; printing elapsed (real) time for each:" -for i in {1..5}; do - echo "Trial pt $i:" - # Run the profiling command, suppress its stdout, capture the time output - elapsed=$( { time python ./training_multimodal.py --profile_torch > /dev/null; } 2>&1 ) - echo "Trial $i: ${elapsed}s" -done - diff --git a/end2end/liger_examples/huggingface/requirements.txt b/end2end/liger_examples/huggingface/requirements.txt deleted file mode 100644 index d6d10e9..0000000 --- a/end2end/liger_examples/huggingface/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -transformers==4.45.2 -trl -liger-kernel -triton -torch -torchvision \ No newline at end of file diff --git a/end2end/liger_examples/huggingface/run_benchmarks.sh b/end2end/liger_examples/huggingface/run_benchmarks.sh deleted file mode 100755 index cf4234a..0000000 --- a/end2end/liger_examples/huggingface/run_benchmarks.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -## Benchmarking Script -## Runs the training script with different configurations and logs the results - -MODEL_TYPE="mistral" -MODEL_PATH="mistralai/Mistral-7B-v0.1" -USE_LIGER_VALUES=("True" "False") -BATCH_SIZE_VALUES=(64 128 192) -NUM_REP=5 -MAX_STEPS=20 -DATASET_PATH="tatsu-lab/alpaca" - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -mkdir -p "${SCRIPT_DIR}/results" - -for USE_LIGER in "${USE_LIGER_VALUES[@]}"; do - for BATCH_SIZE in "${BATCH_SIZE_VALUES[@]}"; do - echo "Running with use_liger=$USE_LIGER and batch_size=$BATCH_SIZE" - - for ((i=1; i<=NUM_REP; i++)); do - - LOG_FILE="${SCRIPT_DIR}/results/${MODEL_TYPE}_use_liger_${USE_LIGER}_batch_size_${BATCH_SIZE}_rep_${i}.log" - - torchrun --nnodes=1 --nproc-per-node=4 training.py \ - --bf16 \ - --num_train_epochs 1 \ - --max_steps $MAX_STEPS \ - --model_name $MODEL_PATH \ - --dataset $DATASET_PATH \ - --per_device_train_batch_size $BATCH_SIZE \ - --per_device_eval_batch_size 16 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 6e-6 \ - --weight_decay 0.05 \ - --warmup_ratio 0.1 \ - --lr_scheduler_type "cosine" \ - --logging_steps 1 \ - --include_num_input_tokens_seen \ - --report_to none \ - --fsdp "full_shard auto_wrap" \ - --fsdp_config config/fsdp_config.json \ - --seed 42 \ - --use_liger $USE_LIGER \ - --output_dir model_output_dir \ - > $LOG_FILE - - sleep 5 - done - done -done \ No newline at end of file diff --git a/end2end/liger_examples/huggingface/run_gemma.sh b/end2end/liger_examples/huggingface/run_gemma.sh deleted file mode 100644 index c882f5e..0000000 --- a/end2end/liger_examples/huggingface/run_gemma.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -torchrun --nnodes=1 --nproc-per-node=4 training.py \ - --model_name "google/gemma-7b-it" \ - --bf16 \ - --max_steps 20 \ - --per_device_train_batch_size 24 \ - --per_device_eval_batch_size 1 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 6e-6 \ - --weight_decay 0.05 \ - --warmup_ratio 0.1 \ - --lr_scheduler_type "cosine" \ - --logging_steps 1 \ - --include_num_input_tokens_seen \ - --report_to none \ - --fsdp "full_shard auto_wrap" \ - --fsdp_config config/fsdp_config.json \ - --seed 42 \ - --use_liger True \ - --output_dir alpaca_finetuning diff --git a/end2end/liger_examples/huggingface/run_llama.sh b/end2end/liger_examples/huggingface/run_llama.sh deleted file mode 100644 index b6a1fc7..0000000 --- a/end2end/liger_examples/huggingface/run_llama.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -torchrun --nnodes=1 --nproc-per-node=4 training.py \ - --bf16 \ - --num_train_epochs 1 \ - --per_device_train_batch_size 64 \ - --per_device_eval_batch_size 64 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 6e-6 \ - --weight_decay 0.05 \ - --warmup_ratio 0.1 \ - --lr_scheduler_type "cosine" \ - --logging_steps 1 \ - --include_num_input_tokens_seen \ - --report_to none \ - --fsdp "full_shard auto_wrap" \ - --fsdp_config config/fsdp_config.json \ - --seed 42 \ - --use_liger True \ - --output_dir alpaca_finetuning diff --git a/end2end/liger_examples/huggingface/run_qwen.sh b/end2end/liger_examples/huggingface/run_qwen.sh deleted file mode 100644 index 54a157f..0000000 --- a/end2end/liger_examples/huggingface/run_qwen.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -torchrun --nnodes=1 --nproc-per-node=4 training.py \ - --model_name "Qwen/Qwen2-7B" \ - --bf16 \ - --num_train_epochs 1 \ - --per_device_train_batch_size 48 \ - --per_device_eval_batch_size 64 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 6e-6 \ - --weight_decay 0.05 \ - --warmup_ratio 0.1 \ - --lr_scheduler_type "cosine" \ - --logging_steps 1 \ - --include_num_input_tokens_seen \ - --report_to none \ - --fsdp "full_shard auto_wrap" \ - --fsdp_config config/fsdp_config.json \ - --seed 42 \ - --use_liger True \ - --output_dir alpaca_finetuning diff --git a/end2end/liger_examples/huggingface/run_qwen2_vl.sh b/end2end/liger_examples/huggingface/run_qwen2_vl.sh deleted file mode 100644 index 963600f..0000000 --- a/end2end/liger_examples/huggingface/run_qwen2_vl.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -torchrun --nnodes=1 --nproc-per-node=4 training_multimodal.py \ - --model_name "Qwen/Qwen2-VL-7B-Instruct" \ - --bf16 \ - --num_train_epochs 1 \ - --per_device_train_batch_size 8 \ - --per_device_eval_batch_size 8 \ - --eval_strategy "no" \ - --save_strategy "no" \ - --learning_rate 6e-6 \ - --weight_decay 0.05 \ - --warmup_ratio 0.1 \ - --lr_scheduler_type "cosine" \ - --logging_steps 1 \ - --include_num_input_tokens_seen \ - --report_to none \ - --fsdp "full_shard auto_wrap" \ - --fsdp_config config/fsdp_config.json \ - --seed 42 \ - --use_liger True \ - --output_dir multimodal_finetuning diff --git a/end2end/liger_examples/huggingface/training.py b/end2end/liger_examples/huggingface/training.py deleted file mode 100644 index 17f3ebf..0000000 --- a/end2end/liger_examples/huggingface/training.py +++ /dev/null @@ -1,97 +0,0 @@ -from dataclasses import dataclass - -import datasets -import torch -import transformers - -from callback import EfficiencyCallback -from trl import DataCollatorForCompletionOnlyLM -from trl import SFTTrainer - -from liger_kernel.transformers import AutoLigerKernelForCausalLM -import triton.profiler as proton - -@dataclass -class CustomArguments: - model_name: str = "Qwen/Qwen2-7B" - dataset: str = "tatsu-lab/alpaca" - max_seq_length: int = 512 - use_liger: bool = False - - -def formatting_prompts_func(example): - return example["text"] - - -def train(): - parser = transformers.HfArgumentParser((transformers.TrainingArguments, CustomArguments)) - training_args, custom_args = parser.parse_args_into_dataclasses() - training_args.use_liger_kernel = custom_args.use_liger - training_args.max_seq_length = custom_args.max_seq_length - print(training_args) - tokenizer = transformers.AutoTokenizer.from_pretrained( - custom_args.model_name, - padding_side="left", - truncation_side="left", - cache_dir="/scratch/jlee436/liger/model" - ) - tokenizer.pad_token = tokenizer.eos_token - - dataset = datasets.load_dataset(custom_args.dataset, cache_dir="/scratch/jlee436/liger/data")["train"].train_test_split(test_size=0.1) - train_dataset = dataset["train"] - eval_dataset = dataset["test"] - response_prompt = tokenizer.encode("### Response:\n", add_special_tokens=False) - collator = DataCollatorForCompletionOnlyLM( - tokenizer=tokenizer, - response_template=response_prompt, - pad_to_multiple_of=16, - ) - - if custom_args.use_liger: - model = AutoLigerKernelForCausalLM.from_pretrained( - custom_args.model_name, - trust_remote_code=True, - use_cache=False, - torch_dtype=torch.bfloat16, - cache_dir="/scratch/jlee436/liger/model", - # These args will get passed to the appropriate apply_liger_kernel_to_* function - # to override the default settings - # cross_entropy=True, - # fused_linear_cross_entropy=False, - ) - else: - model = transformers.AutoModelForCausalLM.from_pretrained( - custom_args.model_name, - trust_remote_code=True, - use_cache=False, - torch_dtype=torch.bfloat16, - cache_dir="/scratch/jlee436/liger/model", - ) - - trainer = SFTTrainer( - model=model, - args=training_args, - data_collator=collator, - # max_seq_length=custom_args.max_seq_length, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - formatting_func=formatting_prompts_func, - # callbacks=[EfficiencyCallback()], - ) - if custom_args.profile_torch: - import torch.profiler - with torch.profiler.profile( - activities=[torch.profiler.ProfilerActivity.CUDA], - record_shapes=True, - profile_memory=True, - ) as prof: - with torch.profiler.record_function("trainer.train"): - trainer.train() - else: - with proton.scope("trainer"): - trainer.train() - - - -if __name__ == "__main__": - train() diff --git a/end2end/liger_examples/lightning/README.md b/end2end/liger_examples/lightning/README.md deleted file mode 100644 index 916d79d..0000000 --- a/end2end/liger_examples/lightning/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Liger-Kernel Example with Lightning Trainer - -## How to Run -```bash -pip install -r requirements.txt - -# For single L40 48GB GPU -python training.py --model Qwen/Qwen2-0.5B-Instruct --num_gpu 1 --max_length 1024 - -# For 8XA100 40GB -python training.py --model meta-llama/Meta-Llama-3-8B --strategy deepspeed -``` - -**Notes** -1. The example uses Llama3 model that requires community license agreement and HuggingFace Hub login. If you want to use Llama3 in this example, please make sure you have done the followings: - * Agree on the community license agreement https://huggingface.co/meta-llama/Meta-Llama-3-8B - * Run `huggingface-cli login` and enter your HuggingFace token -2. The default hyperparameters and configurations for gemma works on single L40 48GB GPU and config for llama work on single node with 8xA100 40GB GPUs. For running on device with less GPU RAM, please consider reducing the per-GPU batch size and/or enable `CPUOffload` in FSDP. - - - \ No newline at end of file diff --git a/end2end/liger_examples/lightning/requirements.txt b/end2end/liger_examples/lightning/requirements.txt deleted file mode 100644 index dbba38e..0000000 --- a/end2end/liger_examples/lightning/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -lightning -transformers -trl -liger-kernel -torch -triton -deepspeed -tf-keras \ No newline at end of file diff --git a/end2end/liger_examples/lightning/training.py b/end2end/liger_examples/lightning/training.py deleted file mode 100644 index ce7cd96..0000000 --- a/end2end/liger_examples/lightning/training.py +++ /dev/null @@ -1,288 +0,0 @@ -import argparse -import math -import os - -from dataclasses import _MISSING_TYPE -from dataclasses import dataclass - -import datasets -import lightning.pytorch as pl -import torch -import transformers - -from lightning.pytorch.strategies import DeepSpeedStrategy -from lightning.pytorch.strategies import FSDPStrategy -from torch.distributed.fsdp import BackwardPrefetch -from torch.distributed.fsdp import MixedPrecision -from torch.utils.data import DataLoader -from transformers.models.llama.modeling_llama import LlamaDecoderLayer -from transformers.models.qwen2.modeling_qwen2 import Qwen2DecoderLayer -from trl import DataCollatorForCompletionOnlyLM -from torch.profiler import profile, record_function, ProfilerActivity -import triton.profiler as proton - -from liger_kernel.transformers import AutoLigerKernelForCausalLM -from liger_kernel.utils import infer_device - -_RETAIN_COLUMNS = {"input_ids", "attention_mask", "labels"} -QUESTION = "" -CHOICES = "" - - -@dataclass -class Args: - model: str = "Qwen/Qwen2-0.5B-Instruct" - data: str = "cais/mmlu" - output_dir: str = "mmlu_finetuning" - max_length: int = 2048 - # for llam3 8B model, deepspeed will OOM with 16 on 8XA100 80G and 8 will OOM with 8XA100 40G - batch_size: int = 4 - lr: float = 6e-6 - weight_decay: float = 0.05 - warmup_ratio: float = 0.1 - seed: int = 42 - strategy: str = "auto" - num_gpu: int = None - - -def warmup_cosine_schedule(warmup_steps, total_steps, min_lr=0): - def lr_lambda(current_step): - if current_step < warmup_steps: - # Linear warmup - return float(current_step) / float(max(1, warmup_steps)) - else: - # Cosine annealing - progress = float(current_step - warmup_steps) / float(max(1, total_steps - warmup_steps)) - return max(min_lr, 0.5 * (1 + math.cos(math.pi * progress))) - - return lr_lambda - - -def parse_args() -> Args: - parser = argparse.ArgumentParser() - for k, v in Args.__dataclass_fields__.items(): - parser.add_argument(f"--{k}", type=v.type, default=v.default) - parsed = parser.parse_args() - return Args(**{k: v for k, v in vars(parsed).items() if not isinstance(v, _MISSING_TYPE)}) - - -class LanguageModel(pl.LightningModule): - def __init__(self, args: Args, tokenizer): - super().__init__() - self.args = args - self.tokenizer = tokenizer - self.model = None - self.profiler = "" - - def configure_model(self): - # https://lightning.ai/docs/pytorch/stable/advanced/model_parallel/fsdp.html#speed-up-model-initialization - if self.model is not None: - return - self.model = AutoLigerKernelForCausalLM.from_pretrained( - self.args.model, use_cache=False, ignore_mismatched_sizes=True, cache_dir="/scratch/jlee436/liger/model" - ) - if self.args.strategy == "deepspeed": - self.model.train() - self.model.gradient_checkpointing_enable() - - def forward(self, input_ids, attention_mask, labels=None, **kwargs): - with proton.scope("forward"): - return self.model(input_ids=input_ids, attention_mask=attention_mask, labels=labels, **kwargs) - - def training_step(self, batch): - with proton.scope("training_step"): - outputs = self.model( - input_ids=batch["input_ids"], - attention_mask=batch["attention_mask"], - labels=batch["labels"], - ) - loss = outputs.loss - self.log_dict( - {"train_loss": loss}, - on_step=True, - on_epoch=True, - prog_bar=True, - logger=True, - rank_zero_only=True, - sync_dist=False, - ) - return loss - - def validation_step(self, batch): - with proton.scope("validation_step"): - outputs = self.model( - input_ids=batch["input_ids"], - attention_mask=batch["attention_mask"], - labels=batch["labels"], - ) - loss = outputs.loss - self.log_dict( - {"val_loss": outputs.loss}, - on_step=True, - on_epoch=True, - prog_bar=True, - logger=True, - rank_zero_only=True, - sync_dist=True, - ) - return loss - - def configure_optimizers(self): - optimizer = torch.optim.AdamW( - self.parameters(), - lr=self.args.lr, - weight_decay=self.args.weight_decay, - fused=True, - ) - lr_lambda = warmup_cosine_schedule( - warmup_steps=self.trainer.estimated_stepping_batches * self.args.warmup_ratio, - total_steps=self.trainer.estimated_stepping_batches, - min_lr=0, - ) - lr_scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) - return { - "optimizer": optimizer, - "lr_scheduler": {"scheduler": lr_scheduler, "interval": "step"}, - } - - -class DataModule(pl.LightningDataModule): - def __init__(self, tokenizer, args: Args): - super().__init__() - self.args = args - self.tokenizer = tokenizer - self.response_template_str = " " - response_prompt = tokenizer.encode(f"{self.response_template_str}", add_special_tokens=False) - self.collator = DataCollatorForCompletionOnlyLM( - tokenizer=tokenizer, - response_template=response_prompt, - pad_to_multiple_of=16, - ) - - def formatting_func(self, example): - output_texts = [] - for i in range(len(example["question"])): - choices = "" - for j in range(len(example["choices"][i])): - choices += f"{j + 1}. {example['choices'][i][j]}; " - s = "Below is a question and multiple choice answers, choices separated by a semicolon. Please select the best answer for the question. " - s += f"{QUESTION}{example['question'][i]} " - s += f"{CHOICES}{choices} " - s += f"{self.response_template_str}{example['answer'][i]}" - output_texts.append(s) - return output_texts - - def tokenize(self, example): - outputs = self.tokenizer( - self.formatting_func(example), - truncation=True, - padding=False, - max_length=self.args.max_length, - ) - return { - "input_ids": outputs["input_ids"], - "attention_mask": outputs["attention_mask"], - } - - def setup(self, stage) -> None: - dataset = datasets.load_dataset(self.args.data, "auxiliary_train", cache_dir="/scratch/jlee436/liger/data") - flattened_data = [ - { - "answer": x["train"]["answer"], - "choices": x["train"]["choices"], - "question": x["train"]["question"], - "subject": x["train"]["subject"], - } - for x in dataset["train"] - ] - dataset = datasets.Dataset.from_list(flattened_data) - dataset = dataset.train_test_split(test_size=4096, seed=self.args.seed) - train_dataset, val_dataset = dataset["train"], dataset["test"] - self.train_dataset = train_dataset.map( - self.tokenize, - remove_columns=list(set(train_dataset.column_names) - _RETAIN_COLUMNS), - batched=True, - batch_size=1, - num_proc=4, - ) - self.val_dataset = val_dataset.map( - self.tokenize, - remove_columns=list(set(val_dataset.column_names) - _RETAIN_COLUMNS), - batched=True, - batch_size=1, - num_proc=4, - ) - - def train_dataloader(self): - return DataLoader( - self.train_dataset, - batch_size=self.args.batch_size, - collate_fn=self.collator, - ) - - def val_dataloader(self): - return DataLoader( - self.val_dataset, - batch_size=self.args.batch_size, - collate_fn=self.collator, - ) - - -def train(): - args = parse_args() - pl.seed_everything(args.seed) - os.makedirs(args.output_dir, exist_ok=True) - - if "Meta-Llama-3-8B" in args.model: - layers = {LlamaDecoderLayer} - elif "Qwen2" in args.model: - layers = {Qwen2DecoderLayer} - else: - layers = {} - raise Warning(f"Unimplemented layer wrap policy for {args.model} in this example") - - if args.strategy == "fsdp": - strategy = FSDPStrategy( - auto_wrap_policy=layers, - sharding_strategy="FULL_SHARD", - backward_prefetch=BackwardPrefetch.BACKWARD_PRE, - sync_module_states=True, - activation_checkpointing_policy=layers, - mixed_precision=MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.bfloat16), - forward_prefetch=True, - ) - precision = None - elif args.strategy == "deepspeed": - strategy = DeepSpeedStrategy(stage=3) - precision = "bf16-mixed" - elif args.strategy == "ddp": - strategy = "ddp" - precision = "bf16-true" - else: - strategy = "auto" - precision = "bf16-true" - - device = infer_device() - with proton.scope("trainer"): - trainer = pl.Trainer( - accelerator=device, - strategy=strategy, - devices=(getattr(torch, device).device_count() if args.num_gpu is None else args.num_gpu), - default_root_dir=args.output_dir, - log_every_n_steps=1, - max_epochs=1, - precision=precision, - ) - - tokenizer = transformers.AutoTokenizer.from_pretrained(args.model, padding_side="left", truncation_side="left") - tokenizer.pad_token = tokenizer.eos_token - data_module = DataModule( - tokenizer=tokenizer, - args=args, - ) - model = LanguageModel(args=args, tokenizer=tokenizer) - trainer.fit(model, datamodule=data_module) - - -if __name__ == "__main__": - train() diff --git a/end2end/liger_examples/medusa/README.md b/end2end/liger_examples/medusa/README.md deleted file mode 100644 index 6d0ba99..0000000 --- a/end2end/liger_examples/medusa/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# Liger-Kernel Example with Medusa - -Medusa is a simple framework that democratizes the acceleration techniques for LLM generation with multiple decoding heads. [[repo](https://arxiv.org/abs/2401.10774)], [[paper](https://arxiv.org/abs/2401.10774)] - -During training, Medusa requires adding \(k\) decoding heads to the hidden states right before the regular LM head \(h_t\). The \(k\)-th head is used to predict the token in the \((t + k + 1)\)-th position of the next tokens (the original language model head is used to predict the \((t + 1)\)-th position). - -The Liger fused CE kernel is highly effective in this scenario, eliminating the need to materialize logits for each head, which usually consumes a large volume of memory due to the extensive vocabulary size (e.g., for LLaMA-3, the vocabulary size is 128k). The introduction of multiple heads can easily lead to OOM (Out of Memory) issues. However, thanks to the efficient Liger fused CE, which calculates the gradient in place and doesn't materialize the logits, we have observed very effective results. This efficiency opens up more opportunities for multi-token prediction research and development. - - -# Instructions to Run the Training Script - -``` -git clone git@github.com:linkedin/Liger-Kernel.git -cd {PATH_TO_Liger-Kernel}/Liger-Kernel/ -pip install -e . -cd {PATH_TO_Liger-Kernel}/Liger-Kernel/examples/medusa -pip install -r requirements.txt -sh scripts/llama3_8b_medusa.sh -``` - -**Notes** -1. This example uses an optional `use_liger` flag. If true, it does a monkey patch to apply liger kernel with medusa heads. -2. The example uses Llama3 model that requires community license agreement and HuggingFace Hub login. If you want to use Llama3 in this example, please make sure you have done the followings: - * Agree on the community license agreement https://huggingface.co/meta-llama/Meta-Llama-3-8B - * Run `huggingface-cli login` and enter your HuggingFace token -3. The default hyperparameters and configurations work on single node with 8xA100 GPUs. For running on device with less GPU RAM, please consider reducing the per-GPU batch size and/or enable `CPUOffload` in FSDP. -4. We are using a smaller sample of shared GPT data primarily to benchmark performance. The example requires hyperparameter tuning and dataset selection to work effectively, also ensuring the dataset has the same distribution as the LLaMA pretraining data. Welcome contribution to enhance the example code. - - -# Memory Profiling Result - -> **Note:** -> 1. Benchmark conditions: LLaMA 3-8B, Batch Size = 6, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 8 A100s. - -## Stage1 - -Stage1 refers to Medusa-1 where the backbone model is frozen and only weights of LLM heads are updated. - -``` -# Modify this flag in llama3_8b_medusa.sh to True enables stage1 ---medusa_only_heads True -``` - -### num_head = 3 - -![Memory](./docs/images/Memory_Stage1_num_head_3.png) -![Throughput](./docs/images/Throughput_Stage1_num_head_3.png) - -### num_head = 5 - -![Memory](./docs/images/Memory_Stage1_num_head_5.png) -![Throughput](./docs/images/Throughput_Stage1_num_head_5.png) - -## Stage2 - -``` -# Modify this flag to False in llama3_8b_medusa.sh enables stage2 ---medusa_only_heads False -``` - -Stage2 refers to Medusa-2 where all the model weights are updated incuding backbone model and llm heads. - -### num_head = 3 - -![Memory](./docs/images/Memory_Stage2_num_head_3.png) -![Throughput](./docs/images/Throughput_Stage2_num_head_3.png) - -### num_head = 5 - -![Memory](./docs/images/Memory_Stage2_num_head_5.png) -![Throughput](./docs/images/Throughput_Stage2_num_head_5.png) - diff --git a/end2end/liger_examples/medusa/callback.py b/end2end/liger_examples/medusa/callback.py deleted file mode 100644 index 673243b..0000000 --- a/end2end/liger_examples/medusa/callback.py +++ /dev/null @@ -1,386 +0,0 @@ -import os -import time - -from dataclasses import dataclass - -import torch -import transformers - -from accelerate.utils.constants import FSDP_SHARDING_STRATEGY -from transformers import TrainerControl -from transformers import TrainerState -from transformers import TrainingArguments - -from liger_kernel.utils import infer_device - -# https://simple.wikipedia.org/wiki/Byte -# For memory, we use binary system -M_BIN_UNIT = 2**20 -# For metrics (tflops), we use decimal system -T_DEC_UNIT = 10**12 - - -def round_to_n_decimal(x, n): - return round(x, n) - - -@dataclass -class Precision: - """ - Precision is a dataclass to store the number of decimal points for each metric. - """ - - n_decimal_time: int - n_decimal_memory: int - n_decimal_TPS: int - n_decimal_MFU: int - - -@dataclass -class State: - """ - State is a dataclass to store the internal state of the efficiency callback. - """ - - n_warmup_steps: int = 0 - total_peak_memory_allocated: float = float("-inf") - total_peak_memory_reserved: float = float("-inf") - - step_start_time: float = 0.0 - elapsed_time: float = 0.0 - - elapsed_step: int = 0 - - step_start_tokens_seen: int = 0 - elapsed_tokens_seen: int = 0 - - step_start_flos: float = 0.0 - elapsed_flos: float = 0.0 - - global_start_step: int = 0 - - -@dataclass -class Time: - """ - Time is a dataclass to store the time-related metrics. - """ - - step: int = 0 - step_time_sec: float = 0.0 - avg_step_time_sec: float = 0.0 - time_to_completion_sec: float = 0.0 - estimated_total_time_sec: float = 0.0 - - -@dataclass -class Memory: - """ - Memory is a dataclass to store the memory-related metrics. - """ - - step_peak_memory_allocated_MB: float = 0.0 - total_peak_memory_allocated_MB: float = 0.0 - - -@dataclass -class TPS: - """ - TPS is a dataclass to store the tokens per second metrics. - """ - - step_tokens_per_second: float = 0.0 - avg_tokens_per_second: float = 0.0 - - -@dataclass -class MFU: - """ - MFU is a dataclass to store the MFU metrics. - """ - - step_MFU: float = 0.0 - avg_MFU: float = 0.0 - - -class EfficiencyCallback(transformers.TrainerCallback): - """ - EfficiencyCallback is a callback to track the efficiency of the training process. - The tracked stats include: step time, memory, throughput, and MFU. - - It requires including `--include_num_input_tokens_seen` and `logging_steps=1` in the training arguments. - - Args: - n_warmup_steps: number of warmup steps - The stats in the first n_warmup_steps will not be added into the aggregated stats - This is because the first few steps might take longer due to jit compliation and other initialization overheads - n_decimal_time: number of decimal points for time - n_decimal_memory: number of decimal points for memory - n_decimal_TPS: number of decimal points for TPS - n_decimal_MFU: number of decimal points for MFU in percentage - """ - - def __init__( - self, - n_warmup_steps=2, - n_decimal_time=2, - n_decimal_memory=2, - n_decimal_TPS=2, - n_decimal_MFU=4, - ): - self.state = State( - n_warmup_steps, - ) - - self.precision = Precision( - n_decimal_time, - n_decimal_memory, - n_decimal_TPS, - n_decimal_MFU, - ) - - self.time = Time() - self.memory = Memory() - self.tps = TPS() - self.mfu = MFU() - self.device = infer_device() - - def on_init_end( - self, - args: TrainingArguments, - state: TrainerState, - control: TrainerControl, - **kwargs, - ): - """ - Event called at the end of the initialization of the [`Trainer`]. - """ - if not args.include_num_input_tokens_seen: - raise Exception( - 'Please pass training argument "--include_num_input_tokens_seen" to track tokens per second' - ) - if args.logging_steps != 1: - raise Exception("Please set logging_steps=1 to track the efficiency metrics accurately") - - def on_train_begin( - self, - args: TrainingArguments, - state: TrainerState, - control: TrainerControl, - **kwargs, - ): - # if loaded from checkpoints, global_start_step is not 1 but state.global_step - self.state.global_start_step = state.global_step - - def on_log( - self, - args: TrainingArguments, - state: TrainerState, - control: TrainerControl, - logs: dict[str, float], - **kwargs, - ): - if state.global_step < (self.state.global_start_step + self.state.n_warmup_steps): - return - else: - # spread self.time, self.memory, self.tps, self.mfu to logs - # logs.update(self.time.__dict__) - logs.update(self.memory.__dict__) - logs.update(self.tps.__dict__) - # logs.update(self.mfu.__dict__) - - def on_step_begin( - self, - args: TrainingArguments, - state: TrainerState, - control: TrainerControl, - **kwargs, - ): - """ - Event called at the beginning of a training step. If using gradient accumulation, one training step might take - several inputs. - """ - # memory - getattr(torch, self.device).reset_peak_memory_stats() - - # time - self.state.step_start_time = time.perf_counter() - - def on_step_end( - self, - args: TrainingArguments, - state: TrainerState, - control: TrainerControl, - **kwargs, - ): - if state.global_step < (self.state.global_start_step + self.state.n_warmup_steps): - # The end the current step_start_tokens_seen and step_start_flos are the start of next iteration - - # tokens - self.state.step_start_tokens_seen = state.num_input_tokens_seen - # flos - self.state.step_start_flos = state.total_flos - return - - # time - current_time = time.perf_counter() - step_time = current_time - self.state.step_start_time - self.state.elapsed_time += step_time - - # step - global_step = state.global_step - self.state.elapsed_step += 1 - avg_step_time = self.state.elapsed_time / self.state.elapsed_step - - self.time.step = global_step - self.time.step_time_sec = round_to_n_decimal(step_time, self.precision.n_decimal_time) - self.time.avg_step_time_sec = round_to_n_decimal(avg_step_time, self.precision.n_decimal_time) - self.time.time_to_completion_sec = round_to_n_decimal( - avg_step_time * (state.max_steps - global_step), - self.precision.n_decimal_time, - ) - self.time.estimated_total_time_sec = round_to_n_decimal( - avg_step_time * state.max_steps, self.precision.n_decimal_time - ) - - # memory - step_peak_memory_allocated = getattr(torch, self.device).memory.max_memory_allocated() - step_peak_memory_reserved = getattr(torch, self.device).memory.max_memory_reserved() - - self.memory.step_peak_memory_allocated_MB = round_to_n_decimal( - step_peak_memory_allocated / M_BIN_UNIT, self.precision.n_decimal_memory - ) - self.state.total_peak_memory_allocated = max(self.state.total_peak_memory_allocated, step_peak_memory_allocated) - self.memory.total_peak_memory_allocated_MB = round_to_n_decimal( - self.state.total_peak_memory_allocated / M_BIN_UNIT, - self.precision.n_decimal_memory, - ) - - self.memory.step_peak_memory_reserved_MB = round_to_n_decimal( - step_peak_memory_reserved / M_BIN_UNIT, self.precision.n_decimal_memory - ) - - self.state.total_peak_memory_reserved = max(self.state.total_peak_memory_reserved, step_peak_memory_reserved) - - self.memory.total_peak_memory_reserved_MB = round_to_n_decimal( - self.state.total_peak_memory_reserved / M_BIN_UNIT, - self.precision.n_decimal_memory, - ) - - # tokens - step_tokens_seen = state.num_input_tokens_seen - self.state.step_start_tokens_seen - - self.state.elapsed_tokens_seen += step_tokens_seen - - self.tps.step_tokens_per_second = round_to_n_decimal( - step_tokens_seen / step_time, - self.precision.n_decimal_TPS, - ) - - self.tps.avg_tokens_per_second = round_to_n_decimal( - self.state.elapsed_tokens_seen / self.state.elapsed_time, - self.precision.n_decimal_TPS, - ) - - # flos - step_flos = state.total_flos - self.state.step_start_flos - self.state.elapsed_flos += step_flos - - # MFU - # 1. Definition - # - # MFU is defined as (achieved TPS) / (theoretical maximum TPS) = (achieved floating point operations per sec) / (theoretical maximum floating point operations per sec) - # Crucially, the "theoretical maximum" throughput only accounts for the required operations to compute the forward+backward passes, and not rematerialization. MFU therefore allows fair comparisons - # between training runs on different systems, as the numerator is simply the observed tokens-per-second, and the denominator is only dependent on the model architecture and published maximum FLOPs for a given system. - # Ref: https://arxiv.org/pdf/2204.02311 - # The benefit of MFU is that it - # - # 2. Implementation in huggingface - # - # current_flos = 6 * estimate_tokens(input_dict) * num_parameters() - # total_flos = sum(current_flos) # across all GPUs - # Ref: https://github.com/huggingface/transformers/blob/616bb11d487aabc231bb230b245c42214ea4b254/src/transformers/modeling_utils.py#L1196 - # - # 3. Derive MFU on rank 0 - # - # rank_0_flos = tatal_flos / n_gpus = measured_flos / effecitve_n_gpus - # rank_0_MFU = rank_0_flos / step_time - # - # For FSDP, num_parameters() is (1 / n_gpus) of the total parameters. So, the effective_n_gpus = 1 - # For HSDP, num_parameters() is (1 / local_world_size) of the total parameters. So, the effective_n_gpus = n_nodes - # For no sharding and zero-2, num_parameters() is the total parameters. So, the effective_n_gpus = n_gpus - - num_gpus = EfficiencyCallback._get_effective_num_gpus() - step_achieved_tflops = step_flos / step_time / num_gpus / T_DEC_UNIT - - avg_achieved_tflops = self.state.elapsed_flos / self.state.elapsed_time / num_gpus / T_DEC_UNIT - - precision_bits = 16 if args.bf16 or args.fp16 else 32 - gpu_peak_tflops = EfficiencyCallback._get_gpu_peak_tflops(precision_bits) - - self.mfu.step_MFU = round_to_n_decimal(step_achieved_tflops / gpu_peak_tflops, self.precision.n_decimal_MFU) - - self.mfu.avg_MFU = round_to_n_decimal(avg_achieved_tflops / gpu_peak_tflops, self.precision.n_decimal_MFU) - - # The end the current step_start_tokens_seen and step_start_flos are the start of next iteration - - # tokens - self.state.step_start_tokens_seen = state.num_input_tokens_seen - # flos - self.state.step_start_flos = state.total_flos - - @staticmethod - def _get_effective_num_gpus(): - # Calculate the number of effective GPUs for the total FLOPs in order to calculate the single GPU FLOP - world_size = int(os.environ.get("WORLD_SIZE", "1")) - - if transformers.utils.strtobool(os.environ.get("ACCELERATE_USE_FSDP", "false")): - sharding_strategy = os.environ.get("FSDP_SHARDING_STRATEGY", FSDP_SHARDING_STRATEGY[0]).upper() - - # Either specified as string or enum number - if sharding_strategy in { - "FULL_SHARD", - str(FSDP_SHARDING_STRATEGY.index("FULL_SHARD") + 1), - }: - return 1 - - elif sharding_strategy in { - "HYBRID_SHARD", - str(FSDP_SHARDING_STRATEGY.index("HYBRID_SHARD") + 1), - }: - return world_size // int(os.environ.get("LOCAL_WORLD_SIZE", 1)) - else: - return world_size - - assert world_size != 0, ( - "WORLD_SIZE should be set to a positive integer. For single GPU training, please explicitly set WORLD_SIZE=1." - ) - - # TODO: add deepspeed support - return world_size - - @staticmethod - def _get_gpu_peak_tflops(precision_bits: int = 16): - if precision_bits not in {16, 32}: - raise Exception(f"Precision bits {precision_bits} is not supported") - - device_name = getattr(torch, infer_device()).get_device_name() - - if "A100" in device_name: - # data from https://www.nvidia.com/en-us/data-center/a100/ - return 312 if precision_bits == 16 else 156 - elif "H100" in device_name: - # data from https://www.nvidia.com/en-us/data-center/h100/ - # NOTE: Specifications are one-half lower without sparsity. - if "NVL" in device_name: - return 1979 if precision_bits == 16 else 989 - elif "PCIe" in device_name: - return 756 if precision_bits == 16 else 378 - else: # for SXM and other variants - return 989 if precision_bits == 16 else 494 - elif "V100" in device_name: - if "NVL" in device_name: - return 125 - else: - return 112 - return None diff --git a/end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_3.png b/end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_3.png deleted file mode 100644 index 34f044faf2fe47d1cf972bb09e9472f0dae877d7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13454 zcmeHud0dj|*EiL)Pg&EHHKkV0XftM(m6jXrwuofq9%7j}rbwwkia=@0RFh_lrleFR zmK&~+3y^K5q?U+?C{!+lfTXerh`cv8GoP9H^(>#y`@GNde%^Qfxw(Lw>ps`H&iS5m zeb3?2eoyx~v%Z@J0)gi2*}dZc2sGIb1ez2xV;b3k-ZD|U@dYJ#CHTo=LLNZU3xKj8kW zlmp-X9DY9k(2te-bRoDstR|53omM<>MpLUsH>Z( z7U%@JIJe_LH)=$8u!DmlDGw*(l@k|y0|<3E>+S;JNT3S!o@W!ye9PsbBR-voorU zxHU>{F;-{spanM=BfhXSYPX5o=-uYxj#wg+#Ty>!ZP%G?b)0 zwxpxQ7u$Cq|4!>b}&$#MGE(+#^V6Gc9B*R;H zjHqh;7@w)tI!7FmM)7_G~1lrS`NxK#hgWz+8$}D+;brxYUHl4hy<*wn! z=1wU22%NOFos?c#hA;>S27OOIKlK}8HtCQKCXdAM5j5|Rp-<0DB&*ZcA&60G3^6Kn zi>$M z0#ZHLO%I3ZRvgVxC8By?rJl$(`%qZ!+m4=lzWZLJ;Jj(j?Wo$Jmm3@$9HKVIsJk(~ zl>4%xU4iXEMSRWd_@F~Gt`D&A`# z2RS%QL-VB$ok|*QBXI9{@es9h#nBpsh4#(G);Lj?uj@QVx7dk`5(RhJ-$c>pHpl z8i!`|(#SEXgI*>amz_f?YPyGoWwdf>*K(}xbr7O+Lrpf?;*ihcNJdfl6&Qg}r3$Rf zLpmM0gARAZE+tu67)I+0Bo420{HlUE!fBB=knXN1y>vbA-u8?6ap>NbG+NF@YB#Z+ zFUPfZ$UA$y@Q2$_5bvOL`uRwAq0DU*%T!}I`fB{4av5?!k)S!PBH{a(p&?O{ij_y~ zUdd{=(vfrQ+mj-ZDfP4?yc!G=WhT{HQp4^1RE$!=X6a1#me~zez1R1>bX~z~e!%J- zYZ8t@kehS3&(O_djos)C*!`0F7pCyT`ed_4N2AaB!^Ye4++*e8E{lXS?PIVxXUYY6WFBVp#A}a4J ztg+G{p(u~5p0rb4G$h>ntH|m!>g#L}f{_K?uv+8PaT|(m_GOqUkLd^3zoa7AdP}DF zLAZRmdQXfHr&i+IQD!N%#pDsr_-h)eE z3lXfl((^AwV1s4=gRJUXlyx;2Krertkb%Qe*0!}+@M_~?Ha0aEluSPUB23hrML!xs zd09R-t6gxp1l4I{ja?YZmU?@`^zj`F9^ew+uHnZ8Lnr~69oCzMt4D0WI-^4p;dwNe zors2!s*4`-luS`JHeOP0IhzMIjDUUF(uWt!D$6Wq#933ot2|I11JiiZRF~Efk5htW z4Q=BB!dN{)QS8pv65G-%ttVsooS@3@j1w0`!5o$XDi#`^fybVmSdGxDe&UmPHpg^3 zG9k1f7IY;w(%}`FMPN10%(v(?^gA;BAvEi`C~yNk5J zvV2`x_zn844ZPbAw^FwTwWCL?$Eb%UblK#-NH+6z2NTV6%PwRcPdY3b)A=OZV>jN; z<1+EfnG1agOR?8v?F!nj3m4e9QXvB`SlNrEGPup`8>O^qh;E2zdB*QrJL-;?oj-8H zWe|ac$CsX}$g%P{gys!T=)4eE8 z$~B%BlIX*P)AW$6Y&$5*uw+M29`57{mZXB_RNk{Bf=#8<&4smkHdueqJ6Yz2w7Xe& zOg!J~xjkYkYjB>b_o&EDJU2CEa>Y*gk;T}`V^Fl}L_-i~L-)`~#hB047s)HXUP-AL z-x4Y{zqYqJaSajdz9lniJ064#Al%TF zRXyw|hzzkdbcSNTVTYD)Esn`PZqkQi*v zd=kzor78-GO?}Av!=P@Z1bt^i);cPp`HFth|J8$`Mq# ze*b#xME-cbot-mJnbl~tk+U#&BAHfK9L6{LTa)N3{g8#>6pA-q$x1)M;2Qn0@cfPuaEH=j4=SYawSgY2uZ6rp10vvMuNYbUEOtzt$VAgal&0I zH9v;Mx-i6AzP-0*LAjEKoe%vb@m5RdN73%9bovoiXW%jP+%t3&IQR%{9CR&@cd1Sl z6winhIVdejA#42y+|7q8s4UIyU(7TQS(FPBi*+qMN39oQGA0(hbxlG1(&4)$2{md*3B^ef-#bEKbkm~gI#YLB&ZU29qaskZBT3v{#SXJaNS9DV5+PpcHOe^zE`C2&JL z5ZD%$Tn}@fG3*{I;S#Q`Yq-dfH9U+00Kj@Ne}hdvwXl1d^WNJQUk}h|SJn{gt>E?A z`fDiv^{-G-TK!hCNm;i7L+s$A(9sBdV)=9(zSnf7$_KVU&reDY=f^rfJz_Ed5vFDL zLH7(y-&Wfj!-?#YCfH-ohN#uGw$hy!n<%CwRGkjc)3_q7n<-1M!!9e8T<3aO6l@*5 zv6qW->1@wskC^&*`u@}*E3?ef6!mJqq_YxS!+17svS4qC>>_$;jF?6tZ8^MRR$6y> zCnMz5V6W1!tkdm!5v%P6cwJ~_u{$1h+@w~Ue|L=`i)x5j%d%M540pZF%-mX{FHv zqVL6J$FE7c6uKl2i(@rnxkCeV?jpWbeAGVT*q$vmi$B|pQj91q<(lE^ z%vF`Ps>Fx*4fkc^Zh)t#T>UmHq4(whAvLbb;YJfBBG4y~pB_D+d~Nq(fF<@CxZSoZ0G1six!dA~;Qp4`7mfd=L3=>=qQIG^ymo3XAM?qH=-BbGY6Wdc#6i7jbRo7UL1 zGmqYHF1}&5@FQ)t=Pff^Dk73wubSy>TF=nE+FOu4CPWh~j*%KzeCrnJstZ?h^z<=0 z936BR##Ha&P+iL!`ni?11#H>*B(tH?QziT(eLs=*`NBPcz#0$k#M8p#nIWzpRjSho zBS%vf`plCtxXQ;#r-73=D3&D(;=N62V5A=Ii`gngB(EAaocbAAcXK7-$qK^Wn2y5l zJyeq>gHG))ITL=&qPT>{9gbV060*K-OrFOx=)x%ZIVCyM(9yb*l%S?_35gZByYJuC zC2_ngu#%0W=?4v-9cd|=hLLSHCaxCn^ zJ+VQ^Dy5OFbUVsSbr*&j7-c`KG%GI#mCu|y`}QvM_&FT(2JY!2neL7@loZ8|aX+32 zo36JzaeRR@g~fOwXs&4#ZR}$hUbCxvg$8zr z%6<`vLw`)N12)ZCO?O{V}N-@84W6O57)~-fcrnqN<+ih(EVWU86&4F3(PCZek z$dNTXLc|}+iN;lKfu!MI_aeAVMBjj`JPzuFj?@Z-Z4yxM%2|uvY@OY9Ozh6TaS&I% z<{)#dBP1mkK6)O$+YTq*$2R8u43W4c{c!Xir;*p|*(qbMQXVWt)|NU_BFs<-c2gEc zkRkYV!STV1E2epDDA*N-;&{{J)ItIwz9}N;M$p&?(>!8S^&>}pgEh%Re!AYd_4*I{ zTKNJMwE`(*2x@!oO~@;us%u=UqbRoOOVEUsbX6rq1}SJ^xv8sKa%l`@QFvrzp(Em& z65VZL6=-rM>f6r2jTV#}4L?YmwPQKexJ+74$ zSaS7fafU_NfI} zq|%vtI0h2H#N&{Hp*NF(EqROL>$=ThG;W&isVU=H_UR9o1QlAB7~s%AHbzItbdzK- zGV)(#HNi%o*oVug*7hy5@p%C7D-~J|UpS5yjwnRkH5kWcNq{?ykMc#w=DG8CSe+TGm|wupYwqWH<3o7Lw6=$x~Yyko*L zxCk!4(;*F5?HXFv!O#(beh}jeCK%Z-q66YPN5VDbfNXKEcG5?{Yn*;Qo4H#5Gd4OO zN-@-iwPc@4hmQ}ZtCKIgjJ`|4wM6Lr?Jz|-#fa)SHc@cXP5U;5Ou@+_ID8LfD17*O z`k|>FytmofsvSWUMzf9WR^jW*Bq~oT-X`LufT8PykM}76u1nExQYI*7B^6G8vg1#d z04&#e_|&sGj)z3tjyM-(9J=kp{VQwonzJey@n zVg{>3$$=_>UJFA#Rm*5Xw?TxpE>PVS19eImJ*gc5PN4(e$C(7&wb}=+j>0lScqpt6 z#?-Xa7`Fst{f_Fr+~pEkwacq`kjIX{XraY=842APF1~D2yx7r>VBgdnS{8G%@U>z+ zMVQE`Om>2~XJx$`&-bP0JIy*Od1x0wTIr&bmKQ+VdGm{HDjM8l`YlHeGt>U**y%^ zEJ?3-T4^-o3rDEG{%~g8tK`gFcexfU{^%I=20o7AiGz_ATLIp-Hs6fj)E28@vf4}EPKeS?iL%}Q0 zC5F0ZqU8dZz80rbRYda2{&GuNU{uEbl0|C{-+uY>Wj@Ko%<1N$)qZQ+;sE-4eH}fx zFUNP+?Dnpz7%bSr8NIqGV}01e5#YPyiP*YEcgnAe3U^*vlnc1oAu#`~m9uuk>$nr# zVp#Y{f77{n+h(j-aMZQ#3bwB7KyE{t9XuO7*yXRU4XIDAh!wZ~CdlB_0?(A-BL8x$ zA~@4c22jpLfwVP6LG<{~PkLIhs1P_U=Ni#AZlvFvG+r(*`L5pu1Ui$kwk>fg2(%vn zb>Q9l|Bh?hfX2?WpJv`wN9i~D}6FNQuc+Q{o~0#2@c zaOm6BfU;)VI?g}vlTp0#b7{mGKQ5d)8+68h8w+yo=4D_3PgdXSv)cCCEN`M0tjN6| zJPEXR-_KQ-t{w0y0XF@Iimn~If1ft$zkOPpl8_X^?UzxwpzmtL*yC;ntgBQDlyF)< z%R1FTbyWmi?X*6_Kz9S3dHkEOXCbRL0cp`M`#!I8;cZ~)lR(dNKd*I`2k`Olb41(u z0esV*uBuN`M7)1`-WZs*nm6~h zhu0-e{oCC6_r5K(CMDSGpO}no$#g^pO%z9nZkWL?K}&tI*}jGXK)Op)>-vNwc9z8iV#I=sKe|B0{5~_8N zwQ`LeoZ%wtsw&sU0gnn7x90v$x}VgEq1IW#cuA4?)B9)Q{DR`WV1PjCCen^KWme$z z+JUh`V=;(R;A?^sJY)kK{QHJ>&29%+^3))0YdZM`)NQjdUx5IHrC_9h3?a6DcyXm5 zro8YM9>#A@%-?^xXv`??@B8*`;gdga#w?%UKWTj7RGYQHD*tnKb3b@0`A>TEDw+Ie ziC6iZ{j<6J^VmPk`u}SsH`0T<(R_s1h#l5hu?Hry{PvP_`$bos{^aPuu4KgzJQAN<;JG+Gl1+w$4}EwME~t&;ve9s6hF|K~ttH@Y6@&fZLNS$o7i^HPVhL^1^&Ddb&2IM;|5yBj055qYjj?_3FS z*&^ytp0%LLr7x&i-LVOlKu$1$f4j-f`_DZ|Nk*J|axppL^9V=>CcO6`yiK zmah18ujjkZBOpDQfL;6Uvn#*C$3|LyI20YOivIV04~g7bFXpx2(bdv2)_1eh%)&eASd4N+CF8TpLRl9+r^DC2DX<44~Q2+4_+C*MB>`Ye3(2>}075nPw}TS)jKTx=|zx;gwOzb}Q5fA9V)i){D0 zCog-1r{+M=ityVbZN;#f*8V1RtU zO0&K3!!9G0@o+DL&frF|#xC7A-iI_NbC2o#`8{Z^N*3>O2Ar**MFaAa*q2MfJWii+fEiL% zSArDxV~x#`&T?aeW{{@Q*C_&-D<44D$v!Q#ca;wNXPh=EpG4>`f0!;Y+BjDCm}#EHB|fEL;O{9{Ls$I8NB zDSQ~8CK&>aTSb|D+xJJpgK5JXED`)vnYT|CJ)b)L99env_(ybRq>hSs2DH_d2BeqD zN#(xJZZARbA4c%qB>pV}2rzcoO;*8+FUKDdGNhsGS3_|FH zb2-Dj6gYk7iB2z_KBYo7i8nViOygY{9$PfI-Jg|jo+EoGUxq__SYXf zcft7W$$NPY6_#cb{i(G3_BviO-yZ$#^R&9xAH%?GSyVL@*{(PZA}pZIz1&uj8zrCNf~TvRyCCp#nd_!GW8Ec7lO4PRgaKB)r>n^#7tv-qi+aZv zDZQlEF^%T17U-R6vBE}jWD%X_ChsU%$=$^e=M!AQ1B8Vdr8+@9&l!w52K2{r>k+u-7JQFTjT8Ff?tgL?YCHiy^9AcN zqA=`ATn(_Ql(#^M;=i)K{=YNZ+HVGH*dCh<8G{a5Q2#E)fH;&pTkQ`a4C7}tBBEx$hXZlo;5 z3N-IkPKsM5UdF}SW}SMuB)s{~W^o1Eb00h>MrxQL+ zFBlF1{!h>z@Su}SELQU6Jbs#z1!{bm13HsBV(cQAZCAUNKKjCL5TX;v^@9A4-q$2B z)3F6lJ#cKUg($p4dH5)YK8ro;ZLhT!QXfAlZ<8l&;J?l6J=q7l$CRM6oZzMhs2K% zISq1>=5<)BP0-L~L2YFO*M}{A9;@p;5;*Au(3Q|mVdkYzx_Qxk63DVHIf&k4CV%L* z`7}^H$aSoJKPMch-qAH!m2?j>p=T-k>G|0c)fa#)WD~JJ$Dpu3r_rBjp&3V diff --git a/end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_5.png b/end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_5.png deleted file mode 100644 index 61d66f6b0465c4b4cf83013e69676eb0ed4dd128..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13939 zcmeHudstFwyEkP{O-xW#O=FeR=7X$*$-@k9q5fEsmCkQknY|b~pH~qZGN)X7#ZvUR0$1Xy}Y)s_} zM2u$aUhegU7jkY9jAR?B_`IR3taL$x*Eeo?~Wba<1b#~ z9IT4%&R=;f$fRiB;TApC!}eVd)^1#~>_ErrUl*>io;7p-+dNKiRVaa!U)GWgV26lHGKE&!RrdD9Y7nZ29qL2g^XXjk-$kW}% zH3Gg|{mw{za;jgSa<{lk9i`@Wooj?5h_YBnrgq=k>{uqj3;R(lLSxl zvNOZkJGor@nD@)MLr5-iy^wEl8s9cfvR4eG9F;d*Sjf7ur0{}h#u!G;$BZ-kkeqKd zVy+t20eg&U@1i_O4`dlTj9rf{hBy+5+q30seH!rTC0C6mITLm1H2aCZjmqSvyJ`F& zjz&F^H;SI9Nh#KRc%i4ix2Y?r-4G=zv$nLf3?N@{ke4MrGS0sEFiZtZh#bPLBVYX9 z?mMB~S)}*R)(jybm|47WVhotjj(P4q42UDv5IK-M9*B_#Hq=thDbXF%J0BlU5|n!= z35AqtD#cV?gpzc8oZ)?D9*)NsW=C?6d%d}R;X#?|JVi2SK7x_jVp1Fw&5)%`gIvl~0cBIyySRCq_f2pynDc9Upghe4 zMiZIm+(SAY<-MB3&ql?=EP*L6TAlR5xrBw8Zo(|al#D38DQokrZxM-iW_ifEekI~% zLs!Y`PL^zToUGTwU!JYERMQ)fRXh$K=JnVN$pnE41<9Qt<|;>O;2qK6Bc~YBlyfE2 z7<`Sa%R8a$;?RSu;c3*nT`ojrjSJE8e#9upr?mpHd@ZV=L!C(WJZfWTLwDW#YB==Z zVqeV^Ga)AqGnI!~hV}_OFmV2^0HNdqtJy!Q0v57}_I%;pvA8WZHtwxT6<4F$&k(0p zVRx6Ef8ThDqu}Fk#T50FwS$9$AZidm;wI=`miNsFA5>K+-+Typ*iCqnf4bYGPY*Ns zLeLvrg0(K8X|=zY+U&%xlcbg`PKS}St8nvXEZe5$CP6JHG}=-WZ7r;G{5#z+<&cwO zCT$!4UVrS<;l500=}TOfFVAwX>a$ajh+H&kY!sLy7m^82 zhfTqVZ1p&fsL3j39+^f5KFxYiUkuP3pN74tz) z{Z=ntiBLQ~!{)vbC?v@uM{)~>+Sr#CbcF-G#X??=;tmHEDtkDFNpM365~&%7XmTrl znsuhuPuy2saW^n>q&54O@!DJW5;l|0b}YIrc8Tg?Yd<{i zj}#rMQ_wZj^nOX8qtdH-@C~4E@r-w2IAz}*1hG5i7sZU1hHFJIeR8(_j&5%ouM+=< zQd2KS*0ZkHp*2vt>(eftO*6BNx2wrYGauZPcdlAh7?vIS{sFT}Drj+9c%g-tixa2i zvFy4j5x+iHhpJW4j+<%}gINhk_5lq6#)=nul@;99$iti7423UmQYk0&_ddf12T?`e zwhyutdoJH*);HF&w)rX|?xe^_^nb7jZWBq-q#ej=RiAMid4j`Ic%>uOO3L$T9?Gkj z$*UWgNA2u=vq#j7q-mw1^2UW0A&2fcQA17|A9KMFH?yY|I1Tc=y@{vqR7ZOJ!;l7f zi%E$o^utQRk0zd7>PS)d>y6*>q)vp8o+Hi`(oKbzyvE8>lo!g%Jl=bub%Wu1yDi13 zDdhO_AbEa%VXdKjRuImwk{V&?*;$ab2{kbyHU(28fPF3p>~265OLhT`YC?14GmA9r zNG{te+raCUIvZe zGM5AkfKFRQ58_^%&VNzviY&WTil#VZgFjh|yXpt^Z2mW(rQ3+Yw-J zz1Flwua`?CMoM2V1o!6rVFidG8S<_QGe*2ycACtpOBDi7Y9dCHLj~&{eFyb+-0^1- zn0$-Y#Mq^Uq;3P*YbPV|2}erIL)^8NlIP-7OiJ`)PU7M_JW8N6%%W^IA3bu%w(pnS zV<>t2gVyY`A*42QDyCxOffV^=n}yv!rc%c1`y#4zJ#vWrmeeuQXoSQTQpHaQ)C+M51-##UO?Mv$vdA(yPKBh(U3UZJMYmZWS@{zH-75< zStpmm$GuN^h>E7vwk%r+DU=N>JVK-otl7x>0VX-zB#|hqA)9=4;yfh~N_WwCM3SoCSMNSh<%L z#5!W0naBy91CXqPB5gj$5tI$HDxZcI1cdMuSR@qf;e_u)}2@J&msF zDyn=fp4m3##ot~O?aAisdc~k}amTW=Wfg46sC&$u)s!FTM8*K|p11}}D)ZGUiTTQl1xy^6vYDUA?J3PDXwXKSHLhOK}LFru-x=cp@pNi)M!k!B{O) zbiAFi@WlGAicsy&Yva#SHRGoDpYPEMzcdzVD-6U+t?g&y&sDoIls%yc47Hn39^kc# z>U2cPLd$KOY?1cE9K1=%M0jIm5UGsV7?PdM3Xm+ii0msE-~}fhL%%-i5a&)5q2(E& zth#6wdL-olV8Djc9x-I)Pc~U8$~k$mC^mlL;YcJR%F&-qm&-3kGkDAZQodp2Lddna zRK3gc3%QLPPN}fh`*g&DOJqG?hm*wNmuJ8h-JFB;ba5XZIuQ09=~48+Do8=iy`*2e zlatD)&pn;~Ub=mAMBMol!Hw&~fy)QV^NID)u}(8eJk*wQq18x1ENc-KGq(U%ymNHI zii{KEDFqU_)&a|cLdpM0vcQ{HEOJ7sJK>o3Jnkzk>dd1%8dLb!$I5Nt7810ca@dw+GQ9OOU zF;uzij0YgAW9-9Gq(`&wD-ZQzwvOKRc10G?!+>O{sNINqCX#>q9qXNF4JksEKf2sf z0Va#05Rl^lklpRmjd6Q?9N=%Qa0+LOqUc>=4uZPq1zpKE_5+6YmGNmI^O)z!$KL2A z68W=%4(v9)EF^Wxw;|R_>I9n<_l;Hi@CeIG87kHzQ|M@ z#Y;&q7 zl~v3a&GOGWv$wN2m{_0q!=MM6$71a%$!c0(^4`UkeGKKppxb$<(2iR0Pt~}I#CG6n z8^`*d%Q5DXgjuM9;`ooa2qhW3E1fONhaaR%^47|9RzW4z=WIeF95Tqy%9JS1Qu)Id zgX!};*(x$FrZM{&ER#(UN1LNolp&g_8G%qqMYAsB zh{N31fQ!ySxN#3lnLs3k#i1TB+u;7ZJM{e>!c25m_3(r;gF}yCaEor^R+ti8#{1Ij z6e*Y7f65=?jzl_B8NyTk1tpJN(3X#7qER#3;^m^Zz# z5-lOEK?n+AhT~7L1-j2$Pd$9*Z$b zE}MPUn?SbgSY(h|MD$@EL39*AZQSpP=B2l%%2~jne$IkO}h(@e$W=)8HcIf_60J=JAqXy^shnEYqway`w{D$J8PQ#SV3& z6tozfC&F)xq~5%)Qjs8f0xsYm1_itUWdd8D>T7ZMjPSIRZj@O9Bfu#EwwyK9?QoL+ zpgYpYE2uL?BmAf)#gx3+3LJm({aK7TW;7kE@6jcUInoJBDr8NUlZ36?tL$Z ziIzlkxbo)^CTCW8ze|cZp(PLK2xd|F%I@spjU) zXTRL{n_;zq(%@Hi`;8b(8}43IYd^*xt;}>u?h06#uhx)Ppn~&R#<$7cwl^Y`Eos-n zTQg^FPdQY2>BrN2SkHuYkMX_XI{Fh2xxmIg^0ZC2-4N!m)cF*j)J^PWAw3i)_OTM> zoy#Kj%xkOMvqksf`Uie9fD}j0f8Eh#{eryfc@g7_Yc}^IqgFo}9nBir7~uo@u%mwF zvK4jWTx;|TJ=w>*O?x4D1`XVPR73*3jg8Z_ZvUavjC@fo#g(MYMKH)0&ADlL*Y_Pf zMjU@AAlS4?4^>Sc++4jjW<1BJvsve8)dJ9ZVCM5HcOVX$4`<&S#yf2>6&C2`dq-Jh z5q&g-)Pjk+33AY>Q-d7t8wBX5CA#y@-q+qfJTY)AyO_Dr7d`&kiD9mhnCGRFMFE0? zzEel&(s4czCT%D>UbRW}2BJDfQcnrN(#*8O?zZf{^GybV8&8ZErM)M=E#a8A3Zvlj*qN;ZxEzWtFTC?UGqN4GeNS? z3@z~)W#H7|Q7WM?H^V(UJ!UMy*I8?g8IzM21CKjWVI-q)?**66EH=F#yYz9WU@to* zW$FEL32%a;Q8lFG+RsprjH*MmlMbWX+Zt@Y_>&JxT-A^g+*Ns_oFo{0a}?sDP1`Ip zFmOrtMI(=wo!T3iZ6Aag3Bw(<2m2GuUExuTtd10js{_0c$FpJUk<4cAu$sHH zeTP=Y=fH{_sdM#GNLo@&7;BQYADNzhrlU8$9fILh1VVVV_<%c|MK$F)COm~ZK_a1| z2LsXgu(gANJY4p%y2M{U%F7Uo5vEz8C80RWtUxC$M@}Fv(d47kyb;@kgcYE+4W2u6 z%nq?T5=h2*5`fJq$_)ZuL8ATr{q5-mN-uL;vz)?EE$I@yx@k~UPQ-;PuIfQl2NLw_ zK3r*#U4VXw(;srf@BO?gLRisqDxMXbE~GsvM~))tsCb;?_yfI1TGF1$rC*Rmj8~vR znTJa+eRKRi>5)DH;>r{7a1$eJus~u09gC+OPM&QPUCKuy9ECish%UBpB-Oh^o7);M zg2xnG%z6^dPI0w+vcazt3X*)|bA~IpX!11H;9ZM0e>c9IGPYc>&K$Wv>cw)- zEdrd&$%tJ0aeIs8$JWuQnT_QFxy?;jXRS#|ZTW^hcmQ=iBZ1pM_=lhD0M)si2p55G z(Y8ySv7K8>JPfJVv|#-1xmAGajPXRk$ZU<~YxZA-BhSMXTp?{1irSVK(zCxQqL`tY z;BpzYP^9Ua+w*-#RlbvqKsKUdU~{d{6%odOqlREq27La&PTu)e1jViwW7$d}?! z1SLn^mjb^#a~V}X_2H`HJqJm0($GMhK~OKS*~X-a2H*|}k2>2<4+QeFd8%J%TQ_zy zs59S9(Cwe(QWDi!(4nQ$8N{=%9|Ac#==63wyUeAVPi*}%x@s~>b8%|Wmg03Mn-yAP zhCSc-BNizgY!gmzx3EYaq+!8dew%8q8fOT4=;#Q(q`x77(cSGiR$%}u&w*UNmnWJv zZRSl_xNFX;U!AYZW|`}lc}6T2I$_40!08@^p2-p7UbpGmFg~p^q^CTVrUL@qoj;;0 z{B@D*2F|j|3ka$iF}H$dS6~HalMq3tPp>uD`pZgKcj>AkpPGm-N+`d8=-6-kD9~Q$ zB^YNwAW05WX9GEdZCQ-l>kf8IE2Ie!O;4of{0p4YtcwFv)@bQG#^iiQYo&rKE0qCW zkzK8CI{71oxA$lQEo2bGoDBknJJrt(IZ*%YLJ+9kK-c6aGu!WHf=72y$iE&Hrre)31KIm}iOvZ&pt3aUp;JNO8$3s`o`t&Ggs>`ncoJKZy z#s*bi|NEis>}PYkZrXD04z_00P8N9+@mk-hJ8x-js)+|4NY~5{=ahcdWKHan!lMv- z#DV(b6=z&}pPu*cCBl7=Q_wNT$x*T${Q2O|i?e)ReR+;YNsKr+Z?P%3Lz@8W=No_{ za(UEry+1w=xEqCFxY)XE^h{W4cHsT{_YL%%oSdL{&#&lUZAG7#l}7gC<#_x&ud5a* zwR403b(ZQd{(|uAUNS1Ow>*H_z6V&;A-f*{bkpy6LO%3EOC_&Oa&&{o`Wu2mLJB8) z5^Z{3=k25|kN6A(KD4y(SWx*A6Zc#Ize^lP`>L&g{Sk6s`WyB$fU5NiE$!o~LfIss z#BqDUimkV7Lb`8t75e}??uF}T6rHK`gK=uNnR%73ZNoWICdY9xS36h!K@n%`pC`Zm z&){&!2_hG@6|2a&EU1(&XpIW74~glG{~gaS5rbDOTHF$gAl|@&Wk<3 zUG$&EIOnG;d={%CxNJ1-7bVT&_GJ+M)%ScxcYg%mjjuaM^e z;ub_rGuyogFc%Mn`{t}#>k91jYsLNpz~;4Vj|<@Y8N`gg#!^k(3-{GTh&Q1*)nS3h zfl9y^8*}X-umdYoQT631G--8;M$3waBpJ<)`E2!yD_a`~F=rSSiMI$3 zKl%1@EAty;AgsSo$1#K(xS1ff%1VU)TDjyC3Aim-uz1yn9RKC3K0J88?9&-+Z~a`Y zgFj8u**j-?sQc$R(L+yv{&@DqQf7?o_BPVV) z#yRtG7<81Zz?YptJkyq02^5_x;oQ1Pv}&p{N;Vw($=|MJ7;M<})~U&6*W2AqTRxp> zja{lXa5e~QwZKv%a)apE_6>A!glHc%-P~xX_GBJl5-r)Wt7oCZL)Of42tTpr<5_Oe zH^)mOM*1?KYDs$JX7l5f8*U~g={qJRCgQKm(_iqJTzqwr0nW?EMnXx0nZb7He==Zi zM%XosG&Aqt)&pC6ZAlgYl85h|@J0r#Ij<3G83_O!hU{(&v{Hl{tZ9mhO9!|R(Ai;v zjgj}H;LZMQCD0l{LjYsjd?prup?Xtg=5Ijb1bJ({ z58zxj-@I`nS7-Ku-1*CGa;t)w|5kDU0F&>NAYlNOcjsuwy=_PJ$^Vy@``o#CcDv{(jNTn}d?#Tp4j(lcx zU*!Q?s&uzK0fx{wcLlPN4fKf0=0u127ydb796)G2Q~30&aV?7=C7zCY4VW(ai+e?z)0Q9mW2IfDYTC)2!dVX%`pObBAg9~i`%FJlR{f|VOU(feNcD|F( z^*bOyOXQ+23FqCurS5;qGh0>zZRpQhwNv}oEL(vGkj1@w82FnFoO3VD)a&*j9p@YU zu)R)yo;I64Kc*f?17G_sP8S4vd*h#uS1Kb%IpILrP%zM_*{A$k=yQOg(9ss+GOe!G zbcdYlb#-Cc7WDo`xPenv;|?~h2y>nS+7lJwqvQ$K%%lvrs+*Yvu{J~q%EP)x2fi&) z50~J9c&glL^uxOs0`@36vJ57EvD=m%el*bVj5|?PIC8!wNNHnl_{@loHY3I;<>4(G z^PG{FY+N+S@^6jr-72n|8a~Ch*Gx+p1-`1`4!HUm(9wOo`b6tJy!cIJOhrCyGT(!| z$tSFBi?-MMfVLT!5$vn%-7mGM$R`|T5LBtYiq!D22K_vs4H*x#QE!iVt7A0OTJz!g zJOlpD0rBn9V=ei{mxSkQ6~S}AU*wzpx| zy(JF!nSBRk*qrMnz@y(U|JXM^t{TNSs`|~<{kcfFbX3I|eWdM$g@50RumDH((8Fu8 z@M3R#((Dhm2;u$I{D?PD^&0Vi#DP4IPM|04F4B$l1Xz673y<3VB@HIS(V=skO3yfb z94q2j3*mu*VDe3#gIn#A)E;^SC#@`JWrRU%B2;e09>E7G&$#_zfKWhnE+{^j>$Uw- zdQZ9aU3-M~;lpwh_zMXO5*0eVJpz_G4;fyfTV^t;D)M32;8EaUcR+p@wFpf#stuT$m$F2fG z)S_BWq+p@b{rdV&`$WR!BIO-Rl%+=ItC5Y10v&56_q@}#7<`iIlfpi3vmC%K{K*ph zt8Msi`o?cVA8M2=m=C|W`}osePDTNR|16rFu&@%NL6dvG&#=0Bbq~nlqVv(Z(t{Ttk40uN^7{U3~X?*0V!fhacyxHz)%-3d|Bu2WZp7 zQ;&S79)*j_n()M0o|)jfarRIv3Lw`V+SfwR5@U>|50aN!wKqSM6m+~!#W8kNG><>j zCvN}>6^=fLPPa$gS?xc|lGcM3UQ5l4q$t@RGtWS#n$aU5kH)}B!s40^pxl$HeGeda z0{9Ax{MaTDknWQv7qil}eF2?IpO6IjAcY*It=o6R>dcOx8MtoVj+#!#Owf*%N!um+ zrrNmEZCc}}t=iy$77YtF(9wZam|v`)*8#L#j7hbxgCshtCuy}t$}%I9mZTmaK!GLa zXN6{_!6|`o?=*9rv$lr1pzp9wMlse{j9MXhWF8X~*jte;D}2~JVCq?V--?bV961fV zm~+hg(l9;?&l8Rc0zl#yIiO1qKHTGP5A4|H8R2>do1SVA*>_%B77D9wrQIwSW&s)O zuBG=siZO+X`f-Qy!v>JwhHlnn;Ej+_l18meSmVjR7v?%vHNOY0>ViM|*vj2JiqlN< z8797p7Fw-L$kbM%W-z7aGPXQx?_V0Dw KQ@qPB?*9N&HHwq~ diff --git a/end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_3.png b/end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_3.png deleted file mode 100644 index 3b860a887ae3a075c905ae04b444d97af199a9df..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13069 zcmeHuXH=70w{FC3*`lDIARB1{DkvpxK@kvQp@=jQ>5%A_NGH@Ngd#SSDn(ErfV3zA zL3#^8QCcJjkrHA+HpB=?0FwX)5;$+vUCwtrcZ@s6ckUhI-tP~_jQ3q@uDRxX)?Cjs zbMKs$>9#Gqw}3#PZDwbT&VxWAjv&yw;LRI>Z+@p;tpR}!zcVvBdEpwAMUAeM3yoa+ zY%x%@{jic%2|DE|eb?zwgXDGtMJk9hgjp z_$v%ZEx(svWKl8=`4i$EoHTq8{%)id3h&k4-ab8snybA@eHVty<4>uO>RT7eLkJr6 zMKHmP12V+q4&mA(tCG&OrMxv>-gKN=#V^0UDYRsHI8=p94{#F^5y}ES9a5i$A%OA6 zT{1pb1=EQ&-h-vw#=c0+74;}?p@JoBm7PfvkD9+(J+j3 zwT|+(9#eZhHd^4ba_DL=T0xtc$mpzU|}=>D)Xq z{KClPC3_VJ2bZm{_Jz44; zUuBJ#SY{t3cmFE3?0k{dJvLnDM;RCwP{-ku?tRb3<1#Lnxg%g77pQYlIx8+iVPtyE zQufds?WtBY+WzR4^Xjt40xrI~)(CFr$Q_JYaY}tT(Zr~zE_5AzCzeS^Es{M*wZqWt zZJEkWLy*-;NC`OVb3D05z{Sl)f?e)lBWP;V{iX`NNqgtUaQcF|wN(V>VJh5avDS$D z_Ov*hQRMAJTj*{;L~*jo(jcc~cGFzLWRh<~4Z-Nb;1x@mIe1!O)3SWZ4VNUYcl5sY zL#u$t&sPQB4>ZQ?Z?V%bm1CPcz!8!orj)&gE8NI1z*LQuoW&^8GVXZjM=a^=bB$<) zu&I4)f}$Zda)fYG3cPK7PrxvBrxW5QZlN9N1W<;Zo;5=}ulYv$E^q zbP==mu@on#-aR^q(y)1dZ8DK;^8(ms#sfmBehH->q429~oVv6o>+lY{S|?Hmifmuu z`p95>sN6-rz0|P!6Fv}Jxi~kTmzdU`5msfofB0Niwoym9<8viyIHTOl)6?@+$nPd-6V5HM9!@rgHamQp2zcZR|dQ`z+2Dvck;IRa6 zfaj6Mqy{#!N2iYTunAs?(SqS+U%uV_2 z3af`?z$|s_$x&-43C}0s6S7RP49h%oE06BllR?WCi+kV?)iEubG!FdY-twk8P90z7 z@(8gb`KX4BroPl+*HKJYW-GVYm%JP%VJ=$6_i8)5tjF$V(6kB{XQ|nVY0A-uak2*K z3sCKG=y=*Gc{PeMRdX~fRJm_oa>~+(<20vd`iky(FejCi#Fp+-Sf}A)sP8(b@W!Lu zVcE4R%02h|nJ!|p>t)aJ*w7(!^wOQqx=L8lo=o_CTIUy;B@uJt28yJnToO*xD0sR! zGwIX)srA2M`uQY?z}7?WQs!yHfF`=8f;{=j!tcMwgNgFWvFz7BXf)foi;?}3{RaN5XIGgX}mTllQ@I+ z<5m|K!yY8FOAbbPs4~BU%mA-WIen!2?dx0Xjk8m&EISAbMuGyu9=|bG7C2Il_XG;j zIyL0izjE3dKSg(;8yq<5JMy;Ck3J}B;H^S=@_c}<(NxD8UlWX*T6&Nt!lTRbYW29E z!+$n4as;SWBbrNY_}yfPtf~q#`FBr%2XHy;Oxs&xDCh@1e;g1i{4WrC9eQQV{S@ zEujdj)t*Rc%}EZ^r+#UHn{*ty<%rYjCWVMfEr@PaeiS<`&%J%>$sO6+ROQXfFM6Zj zc8=~33muAyxNLE!8~QY)gdMTQfvl07k_vcI-PUR{S9MtjQ3a=fVaaKAq*6UXS@PN& z55u(D;y19x9QZ@#LXXaXdqYhga_iyp(&{m8<5X5Ye`0xMwizs=iH->+!!zNNaYWE4~_FeT1k~mejFX+`{&)BET zBSXA5pH?jH5cD#==J0E)zNVc-JP|ce#hJM+F~xp(#Aj&4#(+eBu=Gs*S&sbxLc6D5 zULOsQWeA=}4b8lzr-t=-c|XEP*y~vgI)OvSWaKHQ&=oE-j=1npdLR`!`W6zYQQ4z} zXyw;Kx!ijKll0l!gxqLJg9~8OEo?5&y=B+_s=(RWhJVykkuHl33@kqCRe-=GIiZh4 zS8v7+pC7^2G7iX)n{C5ioz@fI=eM`L34L~EXumx=G@sNHd(7I}T(TE`Cd%2MN>%RR zeS8qCHOLoZ7n%Vi|Lc@ggX40!YrJ&^k{OUhHxHboLFWCJMse~^D!iBGM;C*Z9pY~L zPrb{3)BI`o-V_Apd5u@pHJhm}EK^mo9|_ivoHib{NxF6B@j+GzVLwIE#WNk(p#;y9 zHqc827u=Q)ebe{dVJ2}BVFGr}Vwa=batp7}E9jQ%9U5dkoZ#_?x59cdLFq)T=uz&) zp?Fmk;+mf*PVl7@LX5i{7Tsm_K*DJgNB+e~f=kRkH77l(#;ESr?giBoPE6>{`-$-( z$f`|yQ3s6?SL8}C-g|>29*d!`MolVDwQlX3+xK&`>|*OIc_iMB7A_~PiF$pOQk==K zG(sPJK2x?{SL8f*cSxbD_L>sUtM*)qfrp~hT2JKkD`rh8@i5~M`O87_hfnkTITc#$ zGd+f-f(v)wW@_`$@Bwamb%f{>8M191Av+T`3%I90z)L(#<1l2p#=X!ly7Xk%;qJ+f zNm2E)akjd&gL7d5bQ9m>ee14gnSBX$jc!X-X`UD?C%1;#QreXgOwR2&Odls*e%$xW zRPVOLb7UA(Q1v~P<1%obxtE>5xE7t3y4wOJ#tCGcyKd2^mtSUlVg}U&g3PL}Yx@xI z3xdat`<}MD&z;dp4N@JA3dJSJo~q-3&#pRe7rLR<9(AdBR*|+e6*B5i(9ihl0w)vc zE3li;0GF>mSjgJL+hMF2O>321tpA#I?M$d^e?-}2k?2Llu_9qu`}I&)7<4q{13FFN~XvAzLM*%QvG;5?eHo8^#(!>pZ=PV)#?e*!;+@QC=pzjH`#^PeT)u zw{OUR65bM0cI)C4Nj)1)hzvsG0)Im@PrOzvP4p&>-!PWxlO6tecw311YS&FMwPvj8 zDR<>Z7*snC`_QMkYeR=!$H3PFd0iU0Z8`A)fHluaGh_KtG}!3bOsbXIjCyeYBx5VK zdW?Bl!wnXsXAaE^Nyp{LORzQeh57dP|9UB}*+1KUn^aKIi4Ms~A4~i{av@m`Msx(p zl5AsCJpPIRb|JSgBChld;pB0DWWhlti&(1P zA_Y&bZD*8a>dkFS_IqqKtZ}(GPgIwk@L95;Z2<9Lgfn8y-Na*$7p&00(c~~9KlhfK z7|#Ff(T7yAXN1@@tw=SJlwj=s+i`B{bdR0}3I=}Kd8zXBlgaeFAM7Plp~!a3mIFoWB&q551FBDy)m*~a5hygba~8hYW1g3olyuCO=J zYfI4{_3+bUqJq@BzVq=T^a@IeUHfg<=zWuMR_nc0-eG+zxfi0zI;snIgIgUnP#U3z zTH$f8q4^b3$LMqz-N1_+<`nd~M)ubf)~R}ygB-liCzjEl#TUzSwT5G=mpb+S?gFge z<4qci%(|0Sp*{1@qf$VZr2sdKKT73%sM?WYs}l72@#aNvg>MSVoN_QRsw#r zwC~Z{xe?lZ975nALT#Nc!sq1RuQlG4KRe`8!weo>fe5Cp^|7^|D4sm=QvUd_();Oa z#a(I2KP}f%`0Qb?ody>)(Kb+Y=pLzG2!{?_fIEk63rar+`M?`?d2RK~uaNL?&HFjp z{+-jug3e)Ss0WVRxhsVf8uX$J>0GsL?)L27uBS~(i;avrpy1T?R+O13fSEfiYnZLv z-VP2g&#K9Gwcf9Pj2$%`s;Dn_*bsL80d5?K6t0XDdo_cBbJtfQ6mYug?a)g_aQXuk z0(eeRw^tdWIrEem#tIN9n)A#yc&*nxSc*Y&NaU5va%1sToud=qN}}Pqw>KkZr+8Bo z_W7R(9`?|}**#r-))ymgt5!}Wo?DbT&F<)33-;ultwZlu5Ok!gI*U;!Q@3Huxg2g- z&ni8C0J}ERXEh=$h2BSyd68Z>`4d;ko4cNZ^wGW`HYz1_bnhCzlx?aggidD%ZnM_f zy!hJNMyjZOd2YwR6xOVY6sJdU<$lovqFHBUM5*aTW|!Xc;b-an2$8cfHm?k?QjhkE zgX3$;1!4bqJe(>iAZAJl9>1&V60=#|e44z1BYKoDuU2aF5E+92TIjj$&}DnlJ<=Ss z%Phz>$OgVDCen&JL9q6iOAfsT?veM29sYX2LMf-j2UH{k;8XVmt`{S2saD>~QOPi{ zCW}5WIKsP=kTXT=^etlk_mc zm{q+_v3k9Q+JUA|gJV+Xk97}Up|$6uirK8H0(z==VuT~yX-c*!Xa94VX+1wBbe@FY zGUHVku426+;h>D`qgdJVnTXgS!4Yr$a^JG4(4Iq&Jix~0Sc2;!;2D$~!Z^8>4Rgta z#mqjaxd**z`^s?1ar8ngy>Lz{4E!Q}tjO&?oQ~^^3CIue*c7EwOGX(O}z5S;f?~6%|yE zO;-x2u>m)3ip|QGG9XCpo?2GXO;yG~7e+Jqh%95p;nD8eW1y_lF&h$;4gB_`C_SYh zavr3JLG_{pme2MZJ7+|xF6V{s)}>DI5)Hv@(Mf95*ghpntKGT+favH*aowqiC+!_~ zKDc6lfp*tZm@53wC3Q8=enlyQmsK(i|*xTs~s*v&tL1ODY;x3zSul|cUk`oUm z)_Yy+>&R6iDdo~*ZNnp00<2TK*0rtQu^T{WN& zF(RT*F2+CcpMD(=tj)(}K7}9;PE!q2A#_+Cl)PeZ@u;xhr0~KRyt4o-PiO>3YugrCAo)V5H zHn^M!H3#?TVfX?1maD5IYY2L5q2g*Wgx?OaaDaxv`zDzzb1+UD9q{t!>3!fZv!;b1 z7yZf7k0}mEUEibTaMR_8NS~}DKK6QxBbP~`!*0XUOI^yzMIdCr4|*|$n!$2*^A$-T^L2VDcE5hN>C>mtmqI_fEEs+|ssI{Pt-QXa?)ppyG(FU+4%9;!Uwa#Cm>!xsux_sfmf)n&6#qMb2Hx*0SVR6F?HVfhKrnv~Z{xsPmkF}MMyff1c z=@d?(jAbC_C>~ z-D%1?K{aQsAY`7xu|vx$8qme6OgPC4x;&6MYS!`UYs{1m`@?*kTlmmX*u5Yr(pR!`d z=~9!?99vzi$ABXyM5kjR{7cBJ0}$3TfaEI!RZn?ZuR1G%zi#XW+#-}xHM4-s#!L8| zvKklg^47w_0cuQtlJTZ~CJN01{gNn55>X&YjGgG|3Yw;d{*AN@=JM{wBo&BNk~2U( z!MZCLDS@W)VeZzGY)&QNErZWt9%oOz{IoKMhcpk-Z-{FC@+%89zB0zVG1olTihAXx z5>^3ROb@Xr^?L$n*{hT z;KS-#ni!ueiKFR|k~Znm_M6sC6*0v7Tcb11R0G3`N&)Z=-q)F-QZtugT!>vhM!@1@Ob`lT69(&ejgAJ3EGTI|HpNM|9NHN&lxorV;X-guEu4_SY8Ty)^Id`?2f#*vL0G12xH=e}4oS{y*o3W&4ZN zs1=siKv7CHtlr-(%K(MUI_l?~1&NMw1_;}~Wdrai8YnRfx0Kx$l`46l=1>BZ5tx1E z8ATgJcX31i4A4UGh74Jp-oiUuQ%T35A}@e_CbJS{bV?e7>5KpTSkq*ZzKNux)|EGB zfHIiO!l9N*rb!s!);gSkE&4o_EEj*jpNvSBN=}go+e3qtG&l z`!sJ-sCX6C`V9Z$A4)Ff{z%}+ot(#Xn=@ls$b z;FvNOo}lKLnwIgqG-m#qxwB!HBLHq@)mM7Y+G4DFTLmikOM`uXf=+VlE=N$$ccA*; ziyRS|F{ID#CJww%?}Z6Fn|7ZTlV{^6`wgpOwh6ejLcKN zxAFzEDw-D*qrqijC8B3#$?qXlhS-XNoP zh(PN6lz(D*2+`)h3<$v1zX$xc4~1c4v*dg1KJm*;g^k>$ohibw7N@lC5UJU*!Tw?$ zCw}{v=YBGZGYRpRB^-7;$$zN?{&zyg=HiAc`uNQ;3Y5AX1-_epQJlX-Ir2}jXG9h| zbm~KUozE?hpHYnDLl$4@rzl^F$=xcsi?`Zh{f2?K?83DH@_zHc^G^9}F$3<~ke5HJ z6aj+d)&Hh4*@*scM$udTR7kSap{5JZRQoc=6f3>yB4Wc2Vex(bpE+Yww?1L}mrB-d zU`>ruj$zG6*4bD&q$9ocztrVApZ?6gbQVbYCqv(6lg)2U0NqSjB_!$3HV_5UT33o5 zKYq+y9K)eV+-cH^p8s26f=l4jLgv3N-B0!pR{>&4Z3K@kY#Jd74%n82_=}5+V}Rmj zDfF9xs!=M_AIJazz$rR^`0-u62m?^?&q%ojytsQw%Y$eE+_n4s3H|$Zm*4dT4rOS# z3;@FErv1X5b357`ldzo<)h|Sh5VCN7Kgi*_5oDqec(o;;?rZ`0wN%@i^qgQT`}wKf zY#p#4ptNfBel767nVwEPVbYZ^jDa2i#{Sy2)I6_iob2%&w-5dr(3WEWuB)@&H){P= z@qgVaBl0%)TU&R3jS`B<<_O{3`H4V5$6?QJK9%{yr{AHkfVuJTsf~%k;1*)w z_nrCoL`UQh+rBcf2}ru-L(_W18n zc7H2`YdB!=e~$VeJK;NYtNv(}f8^f37y0j<{89e>J6!+2$jNOHF}42GCLx8b?KfJZ z9lgr0319Hm?{2FCioPErM}M1?qo|aog_=W!1<*2PZJwcSCb~<#GHt6mP*DCD-+*cM znPThwsae#qhQP~N`d2F+{T3iRw|`(-Vsjeqy&>!G{2>=^a44haV^*X@u z3tZMOmjE2oFEC)TADl;BRzNrEp^A;IZO+wKEQRVpmR_j$V&W;D8_>UceLvs=piy=E zwjZ^q-p>u6c^Wd6VmkWb!P=RXissp~o$@GOJ%F(%{j8F6e+Uc9Lro#R-P!hz{bpI} zY;DJIkSss_sFML(TMiky6kDv*GO^98{mwS!j>{2}<%Ol8%>eNbvOJCPz)OGeh+g#Y zTAHPz12>7uo}r_!LhJl#gwt2w2by}wb~x1cIgydg-U53+@ej#kh#M%N8_O*PmE3E;IoEpI zjRJf|)8`Hu(o;;X-fzb{9QyT2o$I5QG0|C?&CIH=M30cf5Rd-Sjt-Q9MFcj@zZ9(w z9Co|vX7B!3+AgoUl!Y8zE+3U|8~;>$^Jm%;pvjW}X$bn|Bx^yzi?GRQv5M`Ebw0Y; zm}9KP44%Glns?3hb&nm&vZ+fd9b&~pxdp`KG~xW^)9}M*JR!Y6kuL`|T>n{%B&(|QvTbYZ15MmNiWhxtJ&L zpe~VX4Gz0TeW?F5s=I@TzN5s%mdXiv)e#;T+O zX@t)KoX4_GXf8uv&1x*9iR3pm+1wo@TG zqRzB@_)fr2&!p`bPrRu;HV2$ygNi22Vaw-{2hVo zCkhl@I4rims6gDiKDtPwov2IC@m-&*!l^nL+#?*imI20#aWHwzTE$-=NWs*w&0te6 z>eJ)c0YC%%ofrx9KT8cf$IsseGZWgs%+P{|{3zbTSXGLA8*BkE+=hFyeA#rA9XhUq z=k=oF4-h1Jr6w`n26pA+$D75h^;tC`KrCMwH%c||2~@=#dwR41XaCy&qXP z4_r4G-`4caTXvN7UkXD3mj3^6^ZK6@@ekVLH@>{HO=ci_j9a1|xy3p+`{+D>nx0DE;~V+jpDL^c2clg;A=@QKuke~ zgs-iP=gD9D5F_k;4*{BpF~UZ8bf5=1)V^t^^w|K=mQQLP7tH=OJfG8N4X`?Iwj4W9 z1+wt{I<7mV&Tl$$4h0ME^?|1Cx6aK;(*5`_WDd84?7(x2!IobcDtcbDpUcZ-E4!R!00W>d+5G5S|tr&d?ztW7)mOPpT zC_JGZ-F^q+UbRN9oE6QlnkevIT?||VUW*mJ7Vv6^X+pzB^NM?^jH-7jrt(5sYrHW} zW!R*f`x~jmvUtNea)8Z|=#^tAg|=7i_Mo;C_enRU+(K-^x&5Zq%8pXdYnK+Qx6Odp z*kDrR5;fC}zbgD00d6)F%Sdnmq@tr1niiP|`3%Vu@OI-tK5LR03EC;{vJSNVWD*n& z^x~6sHi-jwWbROX>k&V4Rk7A8fZ%7N*(%YgSKd5K`Pi42Cm%8Yf_SWPbtXZJ*wwnO z0qEu*5K)cZa!3KNO&{`yZMROZ`ze2XqzCjc0Qk)YSH?veI6MEZ{D@;E+Q`E23FxE4 REMNi1%-G7P;?%_({{bPkcP#(_ diff --git a/end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_5.png b/end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_5.png deleted file mode 100644 index f7920168225fa4351cf190cd71c78844ae4ff77b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13336 zcmeHuX;_n2*KQmTl`2~6REo4xMMVZDKmwsCO05D#1O#M~$rwUZNHBpUimlWEr~`u_ zpbSA762?RVAtDM&7=$ncB8iF+LK09infZI-+kS1&`<`>XKfZIF?_6Jh@Z<{D^E`X+ zwb#1W+V{P3p`Gh+(T}z$Igd{ z#_-iAkXYT>&AYe$bp4mM`dvG#uJ(aeSrJ$#2aD9+9qZu@nl)3DZ3AZTtv#&pfiE8N zGjei<-rB_auBfQb^eZ{unW6pe{XXUlAo-x%@RO%O-8Dvmq>^^CwyZYl^i({brq0i9rj`*6*NfWH0^l4xb= zSNVOR>J_gauNlVk2brt=Pk37~Vn-U|W2joCfY||&)l4^jd}EAvz|P$&Jfi!Uq00fx zwrhe~jB7R{#uQ@hu;M)K5xm45f7_ZmRdm_}J?QISYEhVcl;pVM^2p^b@<<4ItSQke z$$ID#E4nIdFv8@iPEn}q3ytR~zE)H4D#}**eRy#d-p8i5;jndMWC>$em!>MCOjMG` zk|?F`u^CQt-%~%ny;kO&;nZFa@e;Qh<7}}!F!#Owj?)()PJxFx;MHrM8t+`fVrZj2zJk59U-IuRvc0g9_JF8I9TrM8{lyM z{p;Hb)CgfX@xe!J{EMq?gZ`mabtl@7m9e==N`4vBpROD{O*MUS6fgD1XF{|eGW>^4 zOX(l4(H~^J@hMau!+d;`T4DY|woOOZspv^jomrchs?V=j)mgE5E;_~ZgO)0jCL0O| z1KTq;Blz}eb5C^Ah<4mXTdYiit4yBS!3-?xmR;$FVPH>!hR=Q0#<^KdaR7{p2a84< zDwM<4?1a)!ruDYzwWSo2U{P!>cItjAp^IX^$OA2&vp^Vw@L)sEdNjH>Idg#;ar^9|Di^v#`ap=(!-4 zoGmH@cD8~p9|ru%(I_Hp$Y@BbQB|Pn(jfYtC@Kw87r$kwGQg)hh=xH@`Gmawiwg`1 zCwgDS_c{+&oJy+bBf>~Sx=DmKdKB&L!3gI?vMgV575Usl_OH&g-A2Q;6QSBtYJT#7 z$TC#%19F65sma5Pu-37#b*fVg1OlcW<7$G(u?_J*2wSl4%*`J5^nC=ZHQpW%TeSXG z^LFXLvbyC$6sc*3db+Xl%qWXhPhi7uk&<}%iGDh*hO5(j+EMP=4$~?Mcxoq)OEKS0 zenr;8-PLJ`DO=+&fLE0c0Dl6Y45>Ytfy)-YH0nKoY`nh$I z4M){XP>?VR`)P#UsMn10;>MZpRuz79Yltm&x|(U^r-47VQ{V=C?4QSPW;Z1w_VlK! zN2H>u`&DuR+f=e+q#ehNV{WkDm6^@p>?0S?qJm`PVrQbjeNSE%9g1m5cIYj1ouEfW z#8IF>b3%S)qc160Uq)A^25>j9?TyLFGP#yts67-D{#h5Mb+jJpv!{2%t2?4IlICCU zGOU|oJD0yGID*Ol3a+4jC41LJdL_CN4eoM1f#H`9&b~_IVh1O5>Rw%?HGXne;SHFl z4zRb~;zErurp&788S062*)Q;Kh48hs0{GWLtGw8kJNnRBu{p>v#wmF)DaPW2K!kcR z+&!_cW0!6jxTB1ZN)4EF8f)KAg7KIEX9xMxb33~*3?_WNFcgek!U9y>M`t5QwgbIp z0dyR8Pj^p6_5A2(9iB6cXYCn7-OPEryj#(hm~@_APJX_mKT(!L1` zG`855x#OWpv?^ID=4!_h>KF|T(91&Q^`2xH(C}U%!B6(#{yLZi5(yUte%g^w`!YMbV z5rM37=!)O$(i2Oqif@`3OXLRAUiPugCIKh@fgGGTMnyWp!qAPv9jEYI4yW6FzwlH> z$RGPakD@>k&TM#KO((M!*o+*hs0Paz7G&e*wEYDXWfM8ha4aJ{?s|P#!?O(!q+ZY8 zQM@FpycNEWXmhhSjR)B`QmATqEY5l}`Rs;CSG2g>-k%;rS~opchL6PsYxCgQV)R;N zjZB@DM*F;LZly6B9cg#au8JFUNYm~H;KCeom>3+?6g^F|8vJC`Zud2C!)Re>Q}RTn zzo*`2dOSFHdxU)ijnN1U=Yqnel$z84ga@8+d`C@xgrFp(IkqDVp$iO{>fn4?&yI66 zeDR0%=3q9rbKjjsB&qtZr4^g*sVO==o|WBAwha7XbSB~!nRf(o^QcYVW;0|ZS8JQ` zVzb{-A!J1K*}3`(j}W4vS#q^~cRGe$uN)7|ro%BUPE1~}?HqhG6;zv?C#858CswkD zD8sVu{nj5)yI=Nn*LAdrz~VliB=8q!y8~Nm7sY|sg7Jq6S17i87bIDIxn|i7kx)2z zlU5GJyyig_RbO5+J+SUzi%5C7lNFU0fsP)6d~xNk1l6zhuRSQ-65QB{V?tlDLODzi2`1;~{KEz7mElPC#RXD6kYFHJggE>^?x!%Ylc?9-(7Pf|6XL@%vNK?Kjf)Po#H!%x) zQF(VXh$zeH<@)&J(*;=XOHMI!IRYxhG(QBN(@|1e{Dz!XKjH9-f}i6u+-=fCcado6 zIreWszS}B^3Ga!lGz{Q|uqoc12y@*VGwuDJeP5ef9lvziQ5}%c-79qENVMmf+%Bdt z^BCv*(Que%3X_N9y5970S?iX)--f3uw+Nr~_mTQz6;w@D;^v5=%K?Vu>vkTq`yqQN zd|BQADX(`)t<8zu`naYnmQY7*8F$P0-WP!!lMCs>z*Ir)Lz!b)t80_L-9s0}AmSyr zvWrFVe=NRG7` z_Z`R%$SUdyI8#~+-L^|n3n54`8iH|=C=sc&jd1Q#%2+LY5haR{O9ua9;nv!j>T0<6 zXaoYmLc${=c_5bRy>t~yU~<8!+w(KByk8szra)IHzPf~aXoh~y9JS_|e?xYyVvsHi z%y#sSpR`z)c;`jVgYY3&>k5onY{yHK%0og`WJYM8)6;xIP&DHD=sS5hd=9^@(q0ps zIh;+)kCOMI$}crLpAxR}kTrR@cU;$s!aM3=)SBX@b|n6)I0fvhX?Jn%JNB@IiTYv( zqwq(fO%NMyEWQg3k}Ia7G_q9mk!Zh4(S8@y?k57q0c7JSfyar-K4w+RbDKvT;zNm&9zbxJoSaacbee&i!){;!rn6hlDMKv}n(?)B2Q?1=0 zh)%m4&GJ_EsvbxYY#5CWuaWvUERNWZx&4Obde~K8Lt6QNg>qcG_#SwoCvNlug^rpfy~D9Ium4 z$FQf2Hx^1xgHtyvDXFGRBEP+dUW@iXESNMMdeGC_iI7sTDLc*160--%s-0c&?LFW+ zC?*91Sx?2O+MG70l3nr5Jic=n%19$ViN{5+@UHOn-$lO7G+jXg} zfDKbkwr*g^B^$8iP<}~@-FzL2+=FgWA*0y?PyvnvSM|kW_XXR zq}t|am+?OAeEvCB1tO{R6jO=q-->6P)4ZG5tf-EMxa;d*?GUA9u!-bRc(B6dp-8u- z4vHdP$aDhl3o{!WoG^<%G6w|v`ayk(wJ;(t`oJI&CT~sue9|8$Rk06{@;Wx#6l05# zY-G1B6yrD?vNyS6Z?f_UYVm%yo}Z>rNUOiF2t!sn*) z;tCtZi9IIv;KHtu*Q{}8l2KA$xL9CNDf$2b?1g%5*S&#@YRQJX#|F&72Rcpzz+iQ& z3a>g=nEzOKD7ykq)xeu#Exg&9TJwBG)jOqBGIylx&jB;qf_`3R%sM0@Usrr9S4)?* z75H2qSzQ!1a)}@UUlzKC=z8yq%ttW!m^ll=5$3sf{xRe?aPZQwIWAP?Vpym4exq(! z1su(4X&GFU^@TK7&5ty=tq{HdD3G>KlN?gl-#TGyB`sdjR=EHiI4m*UgI8qZM^xw) zm@&1TpQq;Tw||YJvCU{+SPim>N3MTSxUfE@&rK(*)+ggNCPCN(@VTCQ{G^uVcQ!x3 zuhG6PH%mK})C>eq4U#Q!_S!4;y1z`E(*=G(9N6Ol1Kcw|u5cfdekb(lAgL@n#_`X= z1Eh^ht7nxFk2Y8a|6=9Vx)Y`Si8fd7e_vMK1M!GJz^2&oveQpOoP%CeTig&yDxY!B zF{}o;^=(Z7>~rsmK^|Q9$k+vN3Y>IZ;^rR3fDbq9&X<&Dr>(Y75lxcbzVok^@oNe6 zr~{n^Eb?|4-zq7uq~(n*kV-`A%K8b!PNGSI>AE04wgXQm?tz%*_Fn8$u@919CU#AQ zcHMFDzxMB(+DXxemtio%gI+Ohc9w#;zTtM-AxLN$;4^owLkmK@aLufU8tv*;Nb(WX zv9Oo54nC-Q)=-UAhO>px=?m42xn&h*4$mIP=J1ZRv|#7D(oeD+M!5{GDvI}PaG5fv zi-$s%5}|>4WWjVo(wi{%y6S+fC!O^W3HX&2>=})v1ga z@aI(-$fvt*rWo7{g2+45aT3+RbD6ycHI%(Y&vhO5X$HHt&KZ6+SGL=|`-k}!dlrme zN-^l#2);4G31RRWS$LaW+rj%Hb+d)E^7JR^U$S0^iXoUiYv-2q%PuZTVzj3#SwSSl zJGnA3H|?frL$xk#{^Rra!sgX&p+5_&xj8Tk?ipn*=b+RIaMk3|lBo4R6Lff+4o>cz zS`xZeb@>A>bma3J=>h&1ZvI$(As~_mACd1KvA8?h(hL1j#ar zi@(gC7@R+7b18?RBI4Pdg8+5Xk}&mBNTWYwfWjP?u6gNdbd3I45uA6dkU0D>6gT(6 zf@!0EVpC$Ksxdnps_f5RR8Zy;iW-UHOPF3>F3Mt;+&JKj%V{REAHeeBW&oB(GHFlW zYZcP$kdK*u$e*qt&2%RLNAZ3^s7@)V&8BT1OBQ$2_`OczBN%xTH$Q88z)g;#Cy`(Z zMoA`3ms|a)+4&1m1({M#QRyg}7jEtrKL$4$gJDjKYd_IGA_P9k@yzS*5Tmvfw$x!N z5e%Yw>kV`dFc2JUVf7tS$}m&EHmt}m5rObxdwl$Bv{L+RZRlN;Xf!kK(VNSsdAZ23 zgTd^C`1bWNB?1y1A{uM54@M69LizorG&Gs16~?&a#WhZ3Ny`0c<@VTTS6A~}R5KMg z-YAUZVd+m4c7wnAG-9wE`%>D6#EUSY+PQ2d)5~Qn&#y4Xny#Lc)VZkYpz@iAytHNU zO~DzLaFhl2Obm>s?im2v77i{W50HHP>CLetwe@f|d73wilQ%8sWfoH6#zq}T0=wza ztl*KpOl7iVkU}QFy|usuFev}Jkfng_V`Zo?YWgzZf;Bx0dsUW|J_M@FXj z>dM38DExCnvnWtP@t=G^%E#m@qrjcNCMK)ovu5W**u9uW{V1P`vh-K1zHLdWLkQjL zR(>e-!;`fw)zpw4qmcLWX`RC*{z-rI2_xjq+(q^#;sW{a|{iZ%{ zjJj2v+89hTYuj#iA?}Oq_bz>!AiL)wgr*g*@^acSF$XXjogD8Qv)I|B1-&p;fKB43 zZ4hTsnl}ZXd$Eac!u@zqYUiV&+Taz8iunZLDKE{5jW+*j=^7!Gu8HGz=GIVVaX?bp z>#Y4aPE)kKiW{B?k28Fn@RxDN(vx4EA5R-p>_Kht!WNh|d>kl)_~s8sh&>(N{B&`0 z5V1cy{3V;jWIw1|RXhu9@brm7Bb(+kok*^@Kz__3Sfe(O;p+3Cwgd!8w@{+n3i8$2j#A$WztV^dXS;mVD3pIc zb;ERbQ2i-yJt|4X99k0d$RES1xB8fZtl>0r=L2n;h1BTjdkd(P=iU2a-V z1b3o=@#9;8MIr~h2Ngd?b6c>3H!6i6$1}WuKqcYG#U3aL^tN~)%POdxVzcC%pH}SK zJ+!JBSmP~E$F15{uVr!uA68}kgO{yYGisM9c$$=Y` z&vHB`@>gW1@2+e)4zhLaY1w_Sv9o4k-VbiA^DitZGO*Zp;dAe5P*}>T#w#oCxd9{O zd>t6Kap9JgKx*FnOIF_E-*27R{BPa5RQQq%@UrZXoaL|73*rlP!OMd-fW|!-9zTIL z6@?j~UI*Fah9pex)HOMQ+Vg%JU}yj0Jx{Uw~2fA)=WxsO%?O#$>=jiXV<=gW;u<%Zo_#uy$E^s>RfS?_uG+{`VH$J{;JUY~;e*AUpA|0n+~y79$!Fh)up zro{n_nz=_1C9Vp`0T^M&iQ7I09TBho?9E>uYOO8y2k=->pVDxuuPEX5bg?`JI4-SaY}R8^1PQl`gaaQ1FM$%|X-z=x8%KUMePd;s#%tBnOuWq0ulTP0j2;j+Uz!1#FN7qVFkiT( z?|Y*)%QFCD|K)K2cJz2+yFS5i-8b*=kAL|GY8+WKHt*o#Bg+t%kol;uiZQBeh)_P>iK#UxfA!b`9d5J(Z9vg-?sym(f=6B0Eo;+(49Xq z3qZ?=XIUW*L)B?jNn*QL)~|XdVa@jAzXCOK)(|lm{2Qi6QZHU#@v*NaZN9Cp)F4N$e{r9{7(q}J;CJp z;)`ac?8O(2O^1Pk6UbxJdO9I}?_>@q3fy0h6bw!C(oF0LKAwA{eoGK95*Dog=6!L5 z$?u4MFtFMVK6={|EUM>B9Jscobb;aWV=l7*`CxV8(V73lQ+`!}nMAr)$(aC(@LIX} z{{GbdX0-UAqZfRnc2iX-9T!^!q>s5;ec&XhpLgj8rCyJW?gQ2wd~o07HsH8$y)tYJ zSR?<5kbYvu`AhhIIG0KXiZ5JBdfFl zHazXyo&}tM{wdkCw}j;<%``&0bN4`j);p-aO|-JEr@!L5)xQF_G_3yQ`Ip1#yD!HN z11GtE%K8zZ)oZK*Z!iDK;YUq#;w-2B%$t3U?H~L9Mb^;I`7c@abbN=N1o~IAHN~qx zK!KjR!CurtM*67+g_`G5h*Njq@2=Rl_?yVUN{jRBRVfV#{Qi1?2At-3n=*)!jdh(%qm7!VO!>j-}w65|Ro8tfld00<&ig~>xi1#V(HGpBY z%?2I=9(UW{3NUVG4*uo35A*hlMgJ0uyZ3H^b>^afSuVlo3M}Q-xxMN-NKq@s#~<_dq3CkfpRI}J<;U?&cai%!ZCs>+DcyggNI`o>beQb6)bk_it1 z^3gC;5F!Ueh8<10L*Bzw&9hMTvjoLNc3E%}s+Fsq-I{$^J43EL1g`(6-F}7(ISG z&3$0Cyc6RB(ABeEV3hCyXJftKWZY+SPE2VWbqJ9kXZ#wW z9Bp+wjVW3_X9nf|XFX{}FSqlcTpJm#F97Zvc1)hI3=NrG^p$USq1id5vX1=4$RSjHVB}|T#nCNTr-m%GQ z*`y~MGaX2$YNrL&TG_1vla!HSL0L;WbqGF&m5*f?=+r#9HCrlFwxfi-*qQhwv2D{L zbb%BQ*OYtU>;)eMA>f>K9$o`P-S=$BjH=zfr6zG_6ws3OOvn%P3Kmd2hu`R>$^(7#TpX;&;b`KWFHrb#LQDvhrrjM6%CPdknL5T0!TV~%2IeV(mx-OrDonHkp zu7mx`yF*N5HsG4bhVSjMqOgs7oV}X>w+9N_8tGs>pgMM?6Qu)sPcfaG<<+bz6NR=M zj+x;*P6txB>PdU%wNUlj>BU#D*h|>LqtRe-V)M8ue{V0^Q+nlz`v!Cgu=9TzrDF;J z0tZ0p`|ogFXJicd7=~#x;rzPB`b18FyqQ-D{5k8@4@^Vbxu+{*KcDw83^qS(yBcUk zk5sZcAby$UK$E+ZrX8U^sFa54khIb8`(g9mKN(v1__)LLgJQy0HKRc5F!Ec=Pc7%Y zK)Y_4UVouYSRoGSwPXYb_?p$@Fn(QP?lNc2&xH}8R!8AXk6{se7ONgN^RmJ8UF%mX z#LR}_kRX)@=+*-)Y(D7X@rQeLYB6&y{o}I)oklJYN?PPJ8~rf}suxuBE!fJ(hLD$e zav6p09E_gG)F&oB9FY_RyKy=%jvB|seUd>(fnOB(zOIa%mlG=rS_N7h2}CX5ALxl^ zT84bMswX)U06o17Xv(gz*BRKHcm0Q21gLO%o=zX0a+=MQ_sz;XGtjRu8U|iF#!g@& zRk9=*IZCth!i#o4&|?-_ugB|8tBep?EvMd0-D*~O)uRWHzA$28jO78nY*c0N08Zso zu^MOEXJK>It3kGF(!n#2w(`P8_Ijg2{7_GpSd?f~a=G>JNTW9jTL)zlhUO30jU|k= zXbVCX*`irXLF&9s{H+dCfg#&;@A@@J9CUZ(Nl+e__}Fs}q#t s04PDl?S8;7ZvHEqH2%%Ejp!Ccv(t8(&N3Z=WDPoWz|)PmFD&Ih09F2Fg8%>k diff --git a/end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_3.png b/end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_3.png deleted file mode 100644 index 68d682b6b8600afe2ec618db91bc47340826d438..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13950 zcmeHuc~q0v*KX7n^k)IB11dwLsHh-R1;dOpPJk#10tr)@2`NJu0!eI95fGwPfihPF zWF8G;hzcRrFb4<_NGdT*A;bVlAduv~*!I_co!0%<{qFazdzb&PRzlwQoU_l~`#k%3 z_KCY}ZK?3DUH<}sKnmy2oxTDBt#tr_)`YB=1HKt!hu45W$IhHTeZnpR#vN6ta7LnK zL2|p^Q12REGSQ?K-JQE^OuD-bTq*UQHjF97Z+pMN$5xwEsCL$BP2#|goafuN!OyhX z4#!_Ouzk()@!R6#+HnI{Hu-FJzqS5OEa?3D^RIb4Q7FblE4#5VTr}P&!U(fO{^Li7 z%qC`r2+V3?HPb9AtlDV}2z0z?G;}R6t1tzH?ZBrayL3RHn_uYz(*^zLQvv+_uE!Pl z*`C~&z^~4ojrkM=x_?h@BM7v6=YR177vU8QL0Nd>V<9|$=%iG^By46sQ!*QloI59F zSx9rE$B$-DL79({*EZfr&`~mde|#QN;#_~ld#G-4EFql>aXA(^;!xHwT{QQD68c#N za^MD=z(XUJUR#=9k+NN+d8Wca50gXbeA6^N*?T3~mx3z>+U>xd7F)}N@xU)pT{UEP>PJLdY)pHa z*2BVvJWZ^!DGhdQ#@ij6LK2B@EoAc+$mGX*7=8k~1uXr#+DvN-Arm6fQPB&z;6ve~ zNc+eYKAAG!kxr3Gc`{*IVT4Y=Gs*=;^Up-m8szI!fk? zt|1FTgmfH%qTaOaaoy|in&_r0LboVD^lR%yjHvK5^Au#^J*3Xl#Rv|X7v&NuYGgvy z(F?YjaL>^Oq*bX$ly&bMo07gVgXxhInY0n6Sip&*KKU`;x98Ei+$!G+e8A8ZUBW`|_=c5ll%jK-Lplsh2uI5) zB=WYPS<7hCxuGr$9)jfzl(LImBz{H{86jz>>iLx<3&3TP;dHzpc|(l44RMN}I;NVq z7_rAZ7N==mX$}n=A-XLKxonIf-6Khw0TGO^+}QS~T$+%;)qV zxfjs6zHyqWJ$24#K|f*2Hr6m!E6k^|(;<9>oP%s5f1fQ~C`gCPq#`0Qnyu~PJJ7Qo z%p*NW7Bx+nb%fRZ^zFtsMc0bM9KBKvyS|v!PWmj-y>D4XfKcE&-<=qhw^)1`JpR4+ zvBVcS^A?>JvK6l=>$&~c;T6N+b&5~AG|;*up;RmO!|au>=0|YGtsXd{K~j97E!ky9 zL30p2!n`e{f!Q$b*Ecbhv)nf&Kn2jX%bTipTJ*e@a)?s@qZV0OmBhH=){wok1x%(% zJj$h-+w#irxw>~0xn|LXg^1&C#cAgILbt!Vqo9&T{wW%^Z)H&5q4HEwiobLzTNc?C zoGtDvNW-$!CYdq`6Eg4qLe}ph>Ay#}9J?P#+-EtpP-nHQ7HDN=Hv+@Mzyd_w${fX8 zM^uSJ2h5NQIg_9#yX2GIca^hrIHJ@)DJ+gR}IA$!tB z*LK`|+`heO59)A(1J#-Hc{r0Rq=c%@;zE!5UEV8+SD9av)xsBaISpZFptOK4>IqRH zl_PkTSfj$pTqZ6bzipkOCPLf|_hv42hI^TI3Tt02n9EtwIMs2vo|P*d z5N1iWT$z8FAhB&GBzxI8M4~5~(+fw@962hM*s<0gpQc$L=Zvqvd`Tv(C5}4b%Z}pF zI$L5noLb@-G;|xU&cm^3u~j@Ko%Xu@`R7`4Z3wec<8X1-;MMNhcMk~bO&BZ zrC@*+z#9~i77R}+BeuH_MFi523&kl#l#b+}kqA~If&LuVye_xGw_@zp&_S4;yQ8W* z4cKTHDIvU_T3$*+#E?t~AFL_JbZ(4X&s=PSc0(ggXp#fEgP%hTx3(#mMw zTfho!bCS-2oi%0keUPNqzP8=441wl1!>?Ggt(SDJHu0#mt+2co$&GI)8_Ft(nY~&-<8aY1RjWd(YyvASfDmR8!=j@} z4V^i*>;Bit5VHl$uRAh%af&hh_@u$hY|e5WdwpH4p;cxAl}O? z5Nv5FK4j6ks8Y~;0SNQy2Ct~BX~78~)_g*XgqS?DJIh=|TMw^N2VcsHgHX6 zy{R?9JY5_<6<6%m-QZPxhDcVUPA7a79q@xzE)A)8p}kzrOgB?qZzbrvfap)?F#DM% zTFDJ+gC~|htDj614*MgoyZjsu(CZw7bLcgQ@?0>XD-F8M(%;>TtLYOnpr)5B>8zfr zT?y!_7tXQz?57&SNIA%l7wQ&H(FCW92aCB$Hx! z&$4NV7ac5G+j0pS({E$;>T^rG;pxUvjEe>WvO}={c$6e&;0A8^-rQg)icrSuoo3y8 z=uvD7E`ViQ!R5qi8&qq`PCVanooN_*pfm3oO|WMkrQ5OJ(ofE;>;#p?dTr=3vC(Dm zWciOOY|R=me@Y3vcq-GNFF*w8WICp(3ADbLrjhBEo0kip_qKa=_fU#3TgsW1Z;PbO z4$*x(Cq5@Fnet-_kxOBkhbN6_2q0Oy#$Yfy5v}5mZqJJVPLlMW#)nBE-$7nPjLmbmX|1umD$da zN924e2WjS=hfyh&RBy>O|FCxo&j))7PjGe>G&dwI8pYSC>!osW#?!`vj08+*X0;U} zK}fQ4VTqMRjv`2RVEOLh=*-eTs@6GSwjKS$U*R_|0B6z-DyjLxRiR( z`s<`6?nALWJO$P1xJ}naTk5s##QwH_3s5yW%ldaa*F5BP$wAKTAg!V(en&*kBg&Gn1MAsL8G%+#fM0Eg}5lrz=i zUT&XAZ>_4FMU}nSkFx?ixU$pmRW9>U9Hsvmu3Ehcr`%xMcSkF!?%9V$ix7 z3^9Qywb-(GqNvsK^PpQrm@`dG$fq1uQnobp)*z!&>SNB5H^#28z!1HA0;5e8S68G{L*vCI1Jz*pl(EY|eu1XAPy+;13CVaqXE)323(xNR~?_ zuB5}7NIrHz;2rAjvdQY}rnm9L%3~7#{FWE{x1WjGmQ7U>msli~)!@#xDveDY5qw%g zke(p6T%}64-(!SzFQ5oal;AeHk31x`s6K1dV+7mkF%(g07c*dBjTZ24i)T$?5uC<$NAScw=N;D8O& zOuK(R?(A`}jR`+G5S+MU!PF0XAjf5_{>IOlxrm~0gua45L786oi-p!zFcDvq$g_gE z=(wN(p6rWzlObw5ylnLyG+?P?d)Pa61g$q`>}})KO}gIo(qOIr2Q zj58#N%NW#7@_1InYFf^7t6zlMOFy0voiELu1lz4Z?!#j@#Vo+^$tkW+igLxC5P}H~=>K zOooJa&+l)$y5i$&T96Ctsj<{V0SBkSa~v*i_^cY%EkleKL@7S+I7WnbR~e)GDobe- z#^|jvfpmoU*>m;qD01_jw62KGR-trd!h&hSI}dJoN;2~5+1(a`7?djR@ETJXaKCCZ zlXObVa=fc{#zN3^$M7rMlpqGYk~5SPw5L=%%yXsVjQ)md^fw@o_=lIfazS5i;XqQ%UZ+86ouss?u-(zVbFej1nDfu~n!`^9`S(7kWv z{BL4pGJ7l4T?njZpq;M5^}EW&5t|dRhtb=Yh3Tz}=?ROs`Iw33dTN@&#+yIpyxg_B zQ#($Uui3f$Y~oex#Cv{}Eo5|w3e!YPnV96-H*iaX?urH|NPK@b@iL7 z=)}`AQFl7(?{4Ip8Vlj=w_to3C0 zpVF9~G%VI>39AiBPQl%G&`r+XgEy#%=y83?*b5U^0K0?X5P?FP0@NcXGEEOl z)LZJzMS{|!y9(Xn*xpB(U3qR4h4hpknBoF{B&b~;m}dIv7(sGvQ=W4a_c?)sU9iO3 z!RFIw`m*bXWu*wTKM&QJqw51NqcscEMasL z8%Fj_qHQ?ypj0J&hYCoX9A~unkwtcP=v# zEY{}3h-2~EZ}cR+dXHC2bsZPUw0-*2+aF@p4{BwGUNKteK1KOaE8F>4z+iWJf!^ma z1Iz)0V}F+Drx>MVCIGf{iF4Y-E9T2XU9^BF64>T* z4{(oMWC9nSm48>Uxn#;F1wxpzGC?;PbfQ?OK0&HS()YI2Lj%>t`+8hE$N)NaP>WCp znfp|@B0g6bnC>r5R|_Ys4AYwN^(!XCc@xk6qH7)Ot2SoOqbE$5zPh#H)2JmgUZl-X zQ)I~egUd~q?YuLAy7`NmAXeM>?RV`*yGEwM;0>J^IV&Y@Bo8BMOjoXoco3hvpBqU@ z(bKDCp!|obJTK0>H@lPK8BGqrJ})~*F20S8q7Ki$I-?A=!>}Nj+@&ab+YN-X#qvxw zCEqIDt1kdAm?=81?=a7I_EWhopCx}&>6e{dWZ_bjo8RB=V25;>)vgF!WXKRBDYQP= zGh}J6UgY^|=WgX!LiP#E{4CkZglL@9Q-V^o?pEFa^|pw~j}{!}mG1Old0i;;*K0(`2n)tCx^;~5W9@j(N09?F1@3hQ5WjR@W6 z3Tf)US@OE!Y#r;(IwjNY=&v@*?ktM(^w7T4O(c_r3oh%Ip#vNnte6@F-VA3pqYJ_( zTLKH_i%w8aMZrdF&_TluNX|X-R+@j7u%iR2c8?WHuW?I#IDuu(w&r;X*n8-mX0S3F^A8buZ)xVIU8jbvO_Kb|(O zn(PqREv7Ft4-e0ey?$kBA?_I;Np}#PkXP$VSEeEA*~#OqqQ&)`YvrhG2*xvx5D4>w$pkZx0z4a~I##(=}CyM*oWsS4FXN);f{`eF~wENXa3EA65N10-y zohAR`)sz2=JZK)ofnJNyE65CU913TmDH^e=SIUS|F{3FR(Oj>Jn*71D%S@zmF-7+f zgb~3yJpmB2I=2?&INxs6-S3?h=`~d2d$oS1ijuak*m&vPLkd8u*#n91GC(+9PqL6M z#&xa(fgILPDGVgQ7M@h01b&$#cQvgzv~q-Q`Hkv))u7hRTQcQApo&esDkbX_4RZj3 zwQ7bald>msak}A*NN)rK*a~|*oIc>p0MgBncQ$&Tu@VU6s__)!0Gv2ckw8AX0Z@TG zNVx94;uo&ve1b-S)fJ&~>w3wojd!^^c_#vFqvuDR)ZGUiiPIe4JzS`hV?+{PE5 z4UZUp0?OZ@u>I!Fn);yizrHx+A7AWdz1VeA8B$vL^EkhLEI)NW24eYQCFkX4 zjn}!)k?VuskFQS!>pP@w89i_ONn{OP@q}V&@u{1nyK);to|v4A-HjU<(0Kg=61l!& zM~|zUb*TK*rguHV#-A9~lqs&+uDMIc!(>+GHzZ6fNP*L&?XM;ZaGL5id4GCzCtcX# z-5wCS{fGtH^zrO&wQo%wonm*#4)klha6cX68E`hn|6($L#h&)6Jw0{b(x0>GU3Y`=Cl2>~D(?A)us**0qgnpw zy8qkW@^0yuu-DIR+xQ?tl^@X%FZYWR?l$@@aPCjs+Gv_qoEBjUHK>&1G;qqa8@c%T zX+-nVc<|&8vB_qlk!l@iL(M90x+N3B1;YAVUd-!XqR|KXb6AA-x9%$Oz?p@ejX}=+ zsFgC`X;L0ASn-376TUR|D2&1W_7 z{Mp(EfxP*_`PRJq%_l!(zTf=n&zAl*9sb2&zvj4)Lwz*hhr{z{AN=SR08IH9SHEGZ zk8bgA^!op^TO1_3YCL-H zNrW&A|KZ#ND%>uNu$uDN9(Lmxg5kVcq|J>f6YC(ncMolJ0qGGoRN;2&20q}1k#v!z z^$!qZL9Q51&A0MoqD5RN?oUPRFKFnmFpWydbAMWLF3!9m98Hu6wE=A6uZtL}%A2X~ zGXF<3q*C%&+dg@fQVmD{yQ&=m%1<)%dSF9gMo`;M6DyNH7QlZGc(wr2c<#$p>d-f0ILUPG zb+BkOT$F4yQ_=EIkPHAfnEbbo%cDI*zHR)^;uT=uikV~(M&byw! z!EX8vZ67e}pLc1SLh@mtcD*tM3SPg8^S)ID;M#}!5qc)Z^U~kIsvg&!e+Lsk4)xK1 ze^EVrbc>H}@zE_p|6fkGSDCJ((eoW_py&2b)NH-?jfPO)dVZS<@oSCgJB372k`H-i zqlFLl=BNA(>yajw^ZyTc!YQO6{qKLvd9hAO$Eh|Du1nrrV)(@1^=)OCL)P-h#OEj1 zYusu-x3J;H>I&yxYf?tTnWN6I#xVFQJost@K#wmO2owgU3bJ9>OB`%hJEoKMTwvBl z?5)sG02Uu+h5iFGvHu>E1c=`b=Kwnctaly2F#ov`d{vUWtY|#BJJWQ*qPO`k(IS!} z3jsO}9*>5x4&d+Kffu;ZYo}`!Jk<1}-)^J;eT+^pac@ep$!wiR!BVP=1m9BuZwwr5 zm;>5{o0d9W&HzsOOsD`&VZVrmI~Q9K;XrM)z7`rPJbB%z|Kdj5?QH}G#g;X@I2IE?7t`EvhYOt@dw!) zXcuQwT%^H~LlhXE8r@!wq5BNtflgExC*6k0Csn@G{NSm)l;rA8<6;Q!R}tnYuZp7U zpQfX8&y5u@`0JFKzju<)oLg-ZVvi;0&6-@cl;CPSc9R=R@DcWECbuvg(yMN+K3%9* zqaYggFZY6KV6eIj1rN8+hR?R(xmUC19x;Ro%DfgOizbL3tk1q2HrkkU`)ELh0v(}5 zZ-zHm9RDy#FpnTQCKIYW%avLV^EZku!RQ5R>KGIm-PDaBIP7T1^rAP%;NM<^&3zXv z=(mT|0_vcDgmTkOIe4xcJxnGM>3xu8mZZx(=F}PC&2@_Vwe8CSlH(@cWQH?_(qoql z{xoVIA}2`<_9ZJCDSFxqpvcKun9gUT)-8Bm1$>csiR52HLO3}O z#R`44Qcdn}*twWjbc!f&mkQ{=3co3FzS6%=Xe4>uJbk?`2?fh6@X_hG$;T-J!d5 zGEUHa2ZjAy8siQZm%T8usU19O&V}m-(D9=vPAR+42JOc~0V-i^gd2A5VfM(Xk=_`l z!()Y2HqgniWRbT!1T`*~zeNEtooCGs~rrUthPj}b;?%TZw9I~LDqmYMNBLX%e6TVUC{;+OA z7)tH9s6rX^?B#yEsnzcl_6z+FILf+G9JNX~#@+yW@M*%0hJgtDq=uXaoY0xb}X zCPUfutu59*&1>p_Mw)LFE-!1o2zyl_;D{HDZ_c{ufpS88Zh|iDirNXvJOxOsy@M3s omeha4{ir{H6*e0wy4g+5i9m diff --git a/end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_5.png b/end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_5.png deleted file mode 100644 index 77058dc0bb9a17f51914054e528f072f7160eae5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14364 zcmeHuX;_oj)^?N%T8n6{QlLQ9R$CFMGAUtjz^NFpDr002#1KQ0A`pZC(PFiVKrLk^ zpa?{PFor}30g(eJ5duU4Nk~u`0|^lViDV$}6WgBSIY;|m-}RmEeSduYL#~VG+0Wj4 z?KRx@T5DfF?si!BL&Fb2Adv3SBQ7UEpyl2m(6We?+Q5HajzyC|AiI&HF8fYh^cV5; zs!m76sArG$xvVs}Z%aH0FWdk5CWx5}an_R69h$u?ovf|iW*$@*e*W7166MZ&>y98c ztV15W@*(nI@h1nD1~Wd~@V8%FT;Z3B4zApE=$j8LMzw#zwyKumQY5_L`T|u%v>l^f z#>Vn1vl(Y)c@YerB2Ses5wdG@4ORi`sW>yhmSup&L&>sA4?U@1~g!6_*7&rZ50Ls=s+);>+M$>@)ilKumu;b3;E=*hN% zFniB*t;i&8ddsOL-2&z4Hiew$EbmS2%ZBA=C(NZ4SeGnND{AhS|ClL`7h~IoU)5E{ zLtsD7zG~IePfkI0lsZZWgW)1h9c09AqEXWA>h3V>5*(8{(R+5VCdAd37Q3+5fmM3Y zgp#HjB%_D<^PgGjOlBYw`#W9SJFk!H{cvAtBwysYJF6nY*EY^|nsq?LFu%@V#wj~+JEp>0R_GTg#LdnkE~WBfx*DgJ;gZjF znz-7oWL(D1)DtdV-nlY0YL`nz>se>AMw(wI6yU$qUh{!&M&fVNQg|5Xtd*YQe9r@c zs@yfEJa{{s3xyTLW8^*8oPNcK`NY}-?CRjd-yyj99=^5bAhTGLn9+534~J`E{KJ5# z+DVg)%au6QbemK)G~uMbRPC1WI4VonA;pX12(RPSN`|_;_EP>8c$IX9ms|1R<&Bpf zV;xS?>PXeXKq@vQKeLu%CgkX0v7@d5@)N;Jj~^dtcyZOJuYMvwcj=|&QsG7JOX#rN z)6m!$xg(h4nD5CH?PS{+N~J&fMFhd(!r~jkDsin#Dh1P7#dO{8{{k`|d1zyyEO$pRkXqX>0&C(3xibE-GSg<51 zKp9jma-O@DT7*uJGP80-$F}5>;#Y#+Kt; z$={@Qo<`V;K{0$mhS{R7xGp_YlIbg}p&@frcqwJ@Vok>xCdtBsJ}{>0z$veFFOU7W zq@~9AL=#?838)l=9{*x>z*kFkUbh!FT-ANutl~B!AERW29Y*Nb z1Vb|MKMXARgc)&_JKWJx#>eZMDEwFiQ~4RzgOiNLDY$KOn5vv3l;+CJ=KeJ&SVMvf zm`kzB*$yLm2nIx2#5jKEQEpH@e1;oah-jJ*hwcJ@HQo4`BeaM9db+r2oR1YFhgea@ zKQ&fd&nJiy2yP4+(YC=!v36gXTuEsC#vIaQ`SVq2M!|oo>EN!|I{wz9X&}^eoPfU&lY#>fJR9{QF zs#a%Y(jHM2Gt?w6Of)abD}1vOf7H_|IGvO|THGg%?iO0jxL{BkdWE|uF2meq=J zY6T>dUi5ld@N}F?;ei)x{HF{|r-A+5-wNO{<%q1hu z3@xb-o^^xR^IUW@tvbw~fW$)Eoq96@LO5lUPyNHW*5KEJa-nRj165f=Jp;L7j!vF* z0hExYBrb|pfe(bjh+)_A^>)P`-?bRKj$7eTE%I}E%^p4nOA`uRP+9c33)>4;)f)N` z!)O@W_7Zl){qoG&0oVBSis`+hQIL5DqzoJt+oaCKEoQbwv~*kejG&pqkJ3cYu!jC! zc4x?8&U50NP*avG5zFS2#kB&~3* zWu2QFAtho3ygtoH3%q;U?Q6STW9~Z*v8Qc$-B}?q z_%SOzjBQ5=o4dxRDYTxstZqH_sWc;xSnDxz`*$%NyJKUSt;&Pcmbi{ZO+uUa!jBSd zezCPEKNOQYxa|1w$6bnH~< zqX3^M?DsiG$X=M3i^qE0n=8w^bLUPtQxt2Si_+T4Tp;J7DvNxHOzj*C?(}fraP6rK zQ-WXREdhg^TOr%_i`s%VPA5UVoEN&#W4<#FMpF9RRmYqZ{T5?h4`O`lo5)N$(+XVn z3xT~%-lUi@^M65Uur1mE4rDVp&}`nMRJJfj84sb>^Pdc2NMoWf&e`g5rqhow7)1CS z(cIjaEE(=k>3^IPrHBE8`_3==w+C1*cr;XK-5Rjzm7Ee&a`VNR!x`JfUY)@Y`j<-d zi6+>xDCx;*6qzDYgyo>bL${~qPHw{89;sZKs*K#&b0SOIRWKhHP)Yb^@l=J$;9Obp zb$>@P=hiV&w`VE@c+||ZSH$GS;x;de`&N4a2Msxu0*lmD*Z6Op&a-t`lQaFvyOsL}wFT;OyI8i3ZPznQc?U^wy;?eXV{c$Wg{~2 zq6(y#BEL#WdSzU3?`V`{qi=#@{<4lqkB>2AUJ<(pmDKPO4qibXhBn1*tWG~|*5$&% z)1J<4oAsq_sF;%_v7va3`7J+2a6nd0rlsA-qWlQr9|zZq@&iJOI66#bPcsrFcoJkM zrv1s$CUy-pQ4AsV4!&{7Mc}pMGU5`f`?%y_V!AWr?!d~i)hYCW6{B9?j%j%(Anr9L zU~?bOOfT3h20cZYLy0b34E6l$sPFnbP&qab}x;gh$^QPxfB!v&2 z|He7#2gHT%6j6s&sNkIH_+qE4rR)fHLV#C?T-TMC;Jtxk+HHI_df{uz-0eDd z$)Gj$Qg4M_mHSiWLV!AoDM#uA$wcYte&`@Nr$JJ;oabPR&lhv;MIWUmoM=bEcuw7M zo%}VCyvMq!`0(%E?0aFB9ZV57Rl~>J8`zA|D9$jla*b%ek3u~Y2AtuqQFu|wgxq@s z);z$1o7R!xQbB>6&~P)f+i78th{6R};t)$YE2*As6^m6?r&t#esv6}3(Uwt%?PWD4fA+XKq4n`L_5cmh9m=aq=u4DLPLD5y`@g^RGC~P z{C4r?mH6c@nNM`8=H%RXJ$JXoli$c|f!!#GtTG)b$`pq3-@!XQ8K!d(%370A%4-0Y z;N+ARfahMeE7((80GIK6$!T?3F|6}=VhK0p^gy~*uiH+$d|I4Y?C+q!8?%mR0n;h- zL>rAKUbflfQGIBysrw#joJB!$sLWwI?yK7#!usrmH~Z_P*ID%{YcTYwc~l3PFOH72 zW}`O&7E14o~?FYlTpw*=FMCYi2%0LzUCO_R*mjQ;1%e~k44zi zzJAv5O0w`$$iNnMig`ds;WWkTJLwaB?1m5mRYdilldWMh&Jh#H{qu3d{FKgsKv*iX z+jo+Cg6HH0uHr1)gL9p85y}QH5S>;r;eI@pmuWzT(Wl(&=P;@i{9;w@=VUjkc30D{ zB(C}qp+8cg)>am5_J{n_JnML4__FB!8Vdr01A%!N=o4v=*G{Gu_o+`HJ0Xe6Qq864hG?ZZV;3dN~N~n`A|j} z^CyODJQ;Utu8(KdK|Kl4VDdsjGn3jo1A{`uWEKsD=4N_J;0yyUKVWGt(`eweAoyO6 zpCTl<$Aqd%mc}(Lj_2m)Pw(B9Zs*N}G*xlk1hVtnF`p1erwxU_I%*GSI9CvulTB*a z{qyXA9^&H%_YC$rhgNYO>xXinhcKOk{!yH|j~8D;6<5do4+4QN?3&6HBmHn)BE8#i zuubGqK+E~i<7RWhpwH@fg$bx(Ut<5EGPT;Q!!21R!JPnx+%zs1etu3J^y-*Q$o`PnSuCG#2I137KM{O_8>1c6l|?1%JA2@+!Dxz zG|vRv-w#suU$^}Zzx=l0f1dXyo*BJmEq%sX6--}-cUT(rme2P5t$GY}JHXm|C0h8x zj5(QHi_0mP-dvkBR%UH)Uz&E>gSKQ(U7YEi2Ty!N7{IK9c4t(6%r(qv3awhpXUgKs z4{qsqA}d`oEc|jz5?;S(S0O5CwvMuT24ui5+Alb#3)h?Ee`<&v!w&B-9(drFKU3Ve zRO8lM#&pmlvT#qn%q&X9DM#$t8XEE0&JwlivWfy&2TP?UB9XO*kYmo*Z?oz2A0%LdsLD2ysd+lQkV zpK^Y6S+A(N`U%zuVtT+C7_a%R=z>Sju;V?3R^YClz)g+Y4a<`(8ot8gmKNJYgn|Tn zujC2_~_4Sc8R^4h*McPF35FVE`S5Z<)v{%Xzs*4Eax%cJv5cT-tJNJF=40hhbj7)8N*5w(qdbU z_{WvB5L)@bOhL?eHXI-@5S!dkN1f6e+K65Gt|9oThZ4|FiQ21Qc=QGWNdilna?H{a zRS?aO5omZy5?3nTGm&?4D51a1-(v@Cey?{EZ|3PUfQR?BJAiZ3D_eHP{@fEq?n56%R22{3gWPw*lwT$rq0!5(#m%-GPv~-u#tr9uG+1$#^Q^IM~)ry zv7AOmIlsLmQHfE$&fd=eTd2IDP0#!O(JDBwzc%PbX=ODoKsDv1uX~HOpE198aEi>w zElCQ7FD8YmmsBJEccSHc@R@bFs7{GcHk!x;(ge)l8pR$4mM<}s2otCXco<&+2>+$@ znzQmndvd&kUFvKn>xU-{5^D8Y*>L`WAYrCLPHMC zQ<@ft0(bSiJM*y9+^u%foEwdYq@iKuLeak9f!4vgZ+v3*fY(O*!dMz{V4t}9_wA0D zeV(Y7k_&^`aN=zqx=d3NN!1i2e@RM7Rg9jtoEAkkXD-a&E!8gjZT_nh%hu}XZhTZ$ z+9DsfYxWVAH_xS4-poyC+u-#HQ#2y=i6Nk1+HX=%kO2keAs~`w-_yZCXu=?YxC`2` zFc1)Mx;2Pi2gPMCRn*KmfNQ@8@<5*q@CO_8z{09_2PzJ53LDT~Y^e9P{EI=s{*-Au zKp=WdD1R=ay@i~)fN2<}$I8F+Nc30$Xdf@K^$#uS4-^en-`)K%y%M_<1F&0VdE-f( z7(1Msvis#vB>*h$KE|%wzIx-1ue!rVDJjRA?#U+arn+SYB*btRGTFO`zf6QIGJ{<+ zc6@CJR(#=k4(+9SVY+n2e(v!W=5&BsP6F(z-9ZKI^zEWfm6qMmx+4go;b&wu`3a}` z13gryw=+cyU^Uk@I@=!ziI#3mig)h?GIiy(=gUg9&35JMJbZS$W&g$CRDPZmCwoj% zv$kyQcF(LGn--FoZf4y#**~_RW0|>qD%o&l(leGZR7RlSQ0}o@*tQgN7+r6><`aeg z57V3VcHUr&@QBs9r=Mnq0bC#c;?a-r?z}y4L0>UkC0|JI)&haDKAh2I*$kXc82*oU9da4&3+A7~oNB{y zOUgy~*|S9tkg!~#@MOB=SHDZm;UkCkuiSLy=MwLN=f!>6^zI-Pkhu#QS^DKm*FLF; zo5TgO8m>tj%}f9O$rSskdoPLm5EUre0xNRrKEZaJ=v%j)=ts5H)zc%`AXr7|W?r85 zdLU5X_6K&}tkbG-^wPQ5cB}kbDb7o;=K;;L9Y+pUZ8G<+`n)UOk@|YRApV+1`X5pQ zGS|nW|MLOmln0u)OuCMbRlvKcYv9s zh>7l(QWg=)eA$*{hS$0691FU+_H|Fgp_Shld-Mge*=4!truzFWId*7&AZ=`p67#5b zU3f61FXmpiNH*iX4R_9+nQ~!!P-?4*o-#BUkT0oX-ST6HXLLQCupy)tAOw4kNCyYK zbM{1>E0#Pt)-psc=21K9mfJ15&`k`a^BLT9(~@v zLo1b`$iGX*+IuqTUuM~2xMTj{zierp?nZeb`0>iirbOghY8u~beD=!2RF>`j=Le<@ zH$9AWN&K;Ca?P&&&krMv{y65smi+<$y6xV;JC^^vyo5h&F6X?%hU$(psf3Eqsvdpg zBX5cJA>)aDduYW4;q0>wngu0(_0~k1ogVSieAxcd=6^`-&5D0}UF)yc|NoE-U{txb z$ZzQx=B_%%k;iAn<6XHOUC;;LpUJU;x%oCk!hM?);)^BPyumGYMZM7cf-v*QfZB?{@ic@U6rzDJS$RAoLSOgigWM-z}dmH^mqC=bWbv0{=@ho*meoC zBTWFh@su0ACyX^7^(uOXVZWB6cl| zfQ}n6N*4SxF5us0j&I$+%XshGHm9-fsnTDx&8RPLnDtdEJ1ebc7zNJG$v1C)LJ@_f z;p0m$&|=lQZdNzc$S7} zb*7vBnK*z0MK!CW^N;kKzA?V=c+(rw82hS4$4K(KgFSvaHzYah4f4>&+uE)B<=rQ_ zQK^aQ*=ugU(W8?xRv&f5Dj{I6lZ2d3YQ;xCc>o1gegB>yRq0Hh2_;bpw> z;ee47G>ob%*UoMn`A$fi=%(>vk)VTbK_v~oyKg4bp1yZC<=cVN5s=uqC)=2sZU+Cw zPA`DvybJH5d){Z}WBkB$5xaJq#`z>GFeUJbZhV^v0L&$AN%~&)?;q#^Ppm_7$@4Gu zn;(SZ03ryFj3hmvMT_~?(KAEePUzYLJmd@JvQOTEnHs=XRIwdd)eYo@B1LrrwljT6i#N`=3NFz+DRwh8lkOa(Q&!`TkN8M#QmpaL8ko`=ohRoc0kk z?jv+TuDmO5rUOj-^^0o+1FLg4UCC!!uaW{=ez3+$g)f;nsj=suEf&BLukOC*vm10h z1K+pF-oOiR3tK9n*kZ8!fdi84GRDT(BZjAnC!_UwB zu%;&>Uy>II)LckmRl;M;Zag@N3$!LclJqSrBHxc!OY2wL?*%&Xj?bdYv=RR&?mCD1 z`8J|qxlipm8&oRLKACL=^Zb*z&_A?d54=CH>;D>=TSxD6^R@8?vb{&S6tpE!L?oM= z`;zYU#k@BxmW^9HJ_>B_omq_kc2w(CU3kk6E8tQgyN#e0g3C+u-p^xzqgZ?OFIfG1 z66P4)Lkc=p1l##g3g1-j@|Ie65WD zAY?K(I5-$S{xrL|J`wUB=wEgjT{X@qC^&TJ&ZPAL=8#TEZ{erJZ;x?dBKkHjy z6wtQsa^PJJ`$c|KOmXWP@Ud;t%2X1XDk4Wd-Puu;sOPlX97^7SAR(t07Dz<>JX`f@+yEe7>w6n71f6 z)sYO>LE5Vha}FCvooX1*LELBo`qn2(yqf0Hiwl(5BPrmDvDlq4_DF&o*A7vq8!&sr ztLRPh!@{$Z0-&N88{38Ow?a8>^(sR?gL9jJg1noiIL@hsxi@IiPzJEP-mK^sKkAn- z-JNM!(Lf$(BV=RAAwoC{-pqq5FdWl|0TR|eqQk@<#!=0k0{Uf+2v>yEV*7`rv&+tN zH<5fTORk~_*U4xq@?y_Eoek5wq^bz%`A<*YJq`j#O7}nJ}Bw0jc%smrD1SEd-sP z#N>o3$lR-yvHfLzQ3HNn%BMiRmr?fo>t`sVC=U6WNo**Gf({5*+&XMx5dCvJt2RGf zL)C%|uoXkm!|};6S*P*sJ_HqpAcB@Cv|c0l_V$R8ypQ zFtSzCVElmGQNq<%k{d9q`=?3C_DJq}x80lKMTh`b=cUS6^Dj9G$&kA7J#zp!|L>T(c*PXD9g%Ua(uOlq69KcTtf8NCWWdAbQ~WNUQEh zYVMnZ|HlN|JVv9e{UR^^)iFxJJZJ4O6Q0wUMj0}A5e!E` zJtH19(gYNKjlfl!%LsA7CR@GiM#2i?%TNJK6mT+l%9`_c`}d0GP2t>*_*8&9wzvH% z0#1FaJ3;geuE-GBDn-LfqDZu#gcBY zC2hJ;D>{>N_d{P*{SKDvSx)9h$V3zhCb^$x;Fkk&Hq z6Y_I3S4@(Oe>Sih{BrrL7XAkh_g)mZ2IK&DdE7sN^tb9c=Pix8FMUCi(Zv02YgPhx zkSv7|&_`Di*Vu;+8I;kmvW>8s8cnKQ#TTpm?dN+AFmw6yCG&qZY0NSEm zo8@KAIhrdBns8dU?_R9m`1}o?6ik&>TByFbWPiTdZrP>1<@aNT75q%i zj27RTVBpD)W>Tb}FB0fwu^DGCcx@|)yD+=zVI5+CjJ(u| zUX~GTPYf6!$d*0R!!85W9nkvV_IY6l=-C>es$N>13B diff --git a/end2end/liger_examples/medusa/docs/images/Throughput_Stage2_num_head_3.png b/end2end/liger_examples/medusa/docs/images/Throughput_Stage2_num_head_3.png deleted file mode 100644 index 2595387fd68202192af50ee97afbc464e5721bc9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14905 zcmeHudsvcLyEjcUR#Rp(PFb2-W22qS4yGjqO*NTgYIacbgk_!uG8Iz;Dm$B6Sz01e znp$~2L5fILYDsE}iikjI9sx;3ND%FWJc(04e(EQZ$v2sVg<3;{o|oio+7SR;h~5~ zRfxvd8_v&ZTi&oc>#)&`odGFcx&G+^iiXys8V8VTM&PC2X|?d=bn z+ZPw5kZa*~-Z!QK2q~>zEf81RD`zcqckKTD?Da2aztgCKlX|vsp7#$Ab7?ZF4A$Qz z3+me>Oe>Lf(VEx_9BF$rW=SHL(Dk;(9^mK9{rlBlKp-!#Y0QN{mTl5q3;ufTUkL7O z%=ow6SQ`{G&XZM9a&Fi3MIw}f=>o@|V$%7}%I8mdMN2SZ76_nyi9;2Tu zl6z=dqp9??{GAs5q2A7l8_rabX2bIKuneRcB^zE=cqL=Hd5p{B2Y1P+2 zO^aWP$<7}v=HSM25ZePLyvUg+gS{WH*Eq}dlw)w!LFWsjy=M5WSV@MVsnpL*q2Q~q zjtL3)sh+_b)&@3B+s5`;oqWG{lCM(m_iXZ!)%=8;*MTL=G0!zE9~i1W)(x`#NT$l9 z=;88g7ZhIL|6L?WQ-&=uDpM{ly2y#=DY<-whxd{%39nZ1l;9eRIJodNT3jrYrTHYa%9 zsQ23fck|sL?4C-tsnQjxCH1Vs6BTTj$eh9>oRg*75iXX4F?Y0(GUzY zXVgS4kV2GiF2Qc0=n83~=Uur-DEb;Id(JbMnCge~UGTz=qOdyshzD&s&N`P)ijc8T zapk7*@$t}hn2RLB z+-*3SvE(w>5ITk2yp|~%AYVLy2yzw9zO;EAg^O|XE^n)Dd{;E1fgHdg^P2++QcRy8 zT+Tkm=WwM$iQBqXs!$^`0I^Zi#3ElLC zRUCexbyp=8TPBkx`|*%#w8^Cm(H z3}MQa&cXI=BCS%E8O>9DOi;PxmY7T-RO1MlJTg|l1=;?7r3VV`txZ5RX)R1f3s_Kv z9;&=I2}hB?Fo89XD_W7}R9Yt)=@tSFimwlY_98VitH_P~W5s9NbvJw4B@?$i=9F~6 z$m5O4mQbyS?tD!>3yGU}qR*1fe&PYejFxbD#yMxhCgl|qZpfR(efDTS?N)7vZOu=J zC3D`iqxn@H&OY7^MT+?x)=c79ri{}ae4stt0KblkO2czoDGTA>`e}}OO^BBcgglnf z+fLF^Wz8b?rP(QM{Up@{$<|Gns+!v2?P6IrjKSka6G_4dsC+>@!r-zpI46wZ>CzD! zDL)duW{Db)3>fh^ImI23fKiupItZ$w17B}V7nOf#30jCs;rW8FNm#*$e zV(0k#6+A)SwS$qj%dt4V=rG14_evqh&B*GyJsx3+&)MutejUG{(=q4yB`r(iCB$2= z@hTZ!M-WVxUKNQ)-}~VS`gX+`+PwA26QWy&b4qlN_+2-ZcZ?FF{NaV=S1Yf z$)a$T`5jtIs9K#e=aapC)m_nIngYcU@lVLO5td0y=%illmE#emuKBE4$IzOcA_CEY z8R!=Cs6hY9NbOT&KdkLjy}P5?i{rRvZYb;5spzqHmkd#BOER{SU-K|0ECxG@?TuS~ zg)olWNpgd4T+EO}7CuIqysCF`@reGnE={wI>bH_4I7Ls)n<* zh4#l>oO{RH_6O%HqBI#Snb#=u)jH|{$D-*uCw(h9z+UW2;;f!2(KTCA@ z{;2cTsP4za0^mCj7cfO+S5(1+6eJUNIGrMrq6@_?tH$I3{Xtt4Y$wL}zm&B_SjjbaAm%nrTg-rY`%q*y^+i3 z-5U~JB8X<{=?HNy%&)?}-xre^R*pbRhpIh~bXXBPrrKI|6c`XRSIk6~u(1a#x(XMn6M z?IwoEbD+T%pfytT=LDDKIhFZ4BcPXFHD-PgMBOYW*)m_K&;nE~OFmLrJ)0s?Je{*=5X z_fb*k72#(Dr)L#ZupPGy)dY+%Q%^nAC+vcBUbm1ox{N|W` zg9^zmsYP@6Gd%-nce*PwhcG#$D3K4VM7t*vQPNFSc5L3|-(t?!!@uVV6VKu3q8xZs zwUmP7r!*B{ML{(b66eK0!i=EFU~Kt}pz$rE{qfjy?c}5VxktSF^jUM zAe~L5_F}&Y)$5`6yZs2?iO3d59{&P{DAhH(73B&Gv9PnLbP{E%cO3bBL<5-}T(0@P z!qlNeTg0dw5%#$?Lls!=RG!6AHpeLnO2|#2AqNoDqjq}>T#W9?vX@40OHSBR8nyH` z{JS@6r5$11ZK{e#V*XSEUqX$`%4HWjxpIMQlU?nCm`_o_;jIgn>bpzHRD{7g>9!y# z{u@_G(frbbu^Nny7spLjakQT!?sv+b3?x>kd=*{hqn$)dI6jj=s!QikoE@dFbfOpb z@>@iF=yA9pMG;)mi{tW=3i(0(c^!(u@m9C!tLg)WAwVF?T#XHTL5-k zxfSgdydWH7AEQ4}n6Zp(2c6YWb*q%AdX~*TDOb%z!r5Nv@iIn3)tK$Xjm3(O6FaOF zSfVnoR^t@GMY!@Tt<4|Jyls-dk%Pb-+Tn1%vBBP#A`g=DhZBjh-uN8-<~MbyoVpv^ zNkUs&(FWvraTZs%nJTWLKBUp?4z&-CHHEn%;I3Wkd>WF4c>BjEMn?dPo)V>i{{&Z=SYly0u9rn|Hi&aFSlYun<~1FqwG0$ z&%d@QsBo9`VZIozlGor$jRG=i_NU=vj@mujVYm!(#@D~l^@Fy#$kYhx{M<9Vv;12n ztP;B`l#*`lf+>Bj)PhD|xI9#xp@BIWU69}zciwZGO5?dQ=sT?30AlrHOxw;9m5Skg zsdm=bV8%+#q@q{##v9~1Z3krvR%}1ttHbh|B~)f=m3M>+8PL?tT_mjhR*tq3R%*d@ zYI&C^;?O0+?1Xepf2ob>cC9bkyySND;SBqzktsd9(kp>F5ZaG1Ux7+{iRK~*WQVUg zU;`j`o4sa0AjkLo=WjfI*Wo?*f(g~Q#+~oqf5ph8B|&TcQmQ^=-XhIK&!8g%&IkD( zJ@^!oU|CNrM|J>#Ag!x^%;Fd}3{Q9>ChI~C#Gs<@yE&|r$*|S9G;YH?p=GW)W11B z3>2ZYR>T#qNO~@NqqAqk+oxwGQ2H2j&>27O%;WY=Ly&tKNBv=3i)m=DE-7!cB}9;ISbS5qMbyGs})}m643VMNUd7bPxA+T0>D2 z>MBxQ#*ZXjxBG7RaZdhfW59;YU#R`+wzFqsOVjN^3qn9y*DV&PJ;ZB_HxeEQSNcWG_E6>qTJbzdJKCW3QIIM{b{5qllBjqLV55=kr{snugu9#0*p^h^*><*W#)4S@dV-%~#$Ru0df;5yy|AV?sDNgn5b>4B z5Vq2zEbws1#O;F9J|WNy?V}SXbBqp}MY9>Y0-=8rDaJeO<{P91Gq8MBK}qO2gJ_{2 zDXXhIwBJU66R~i4Aup=q^DC!%&Cpg!a1qNaDvNN@ZrqimY6M$bvfjsTOm3 zFa9jmFOrY!QNE~h^GyTW2at19J@Q8gXt%wg{_$i3Ur) z`&vo@hdO!>Wvzh&Wvs+4pM`v;7)_?Tr+OblXLn@hr6PNz9JE>E(IQS+C+@B?zj)LN zTKr)o0bjXAR!iO`1>9i2 z^-*nJT?W`hMDCDvE)mtcdoHTOh~M)n*o0IMm2H zRwQ0kAuMKuwLNUh(O9$5VAj)@Y9ln6-<6oAXx}=>)-*HN0;dtVICE;|l9xsa19p4z#KWg7XcGr7F$u7(CxjS`{Bid*1pYtLscWNS!ueQP=2#OPPK+9Q z6foRX4k#yNTzIrpXagMB)>SCL^aRviTN3c>{G5Qs)U5$MwQ;*VXF(wAzb&?OVP-ox zfCp@?1*vCjRwVkW8Zt18M5a`Br zTJ>t^>5nI@M5Rn!mUFJljr{VylonHEg8qe03K|g-Sr@M#(4Kqb*Ri5Hy+oa5YdlW9 zyAUTB>u)mSS)^`@JXtZ29@_$(pq{|kF6Gwbl7C9 ze59w2Ihal2J%Rbu-wFd9V0W?pm13iVyF9XmE94CXOYNPrm!Fr!j?XEVJg6?FXqeHb zM$_K~)WjbnhSV;J6O{Nh*{cbab)2TA)UBZ}1t5s;*(udvRAn9IRYum#8Lo#N^+#*E zhL#3r+PyCe^clgr$1Vg6!HAbK%G4IqP>`PiVK;3Ay*Pf8qL-RuezX9&PU2 zGaGVf&M!~X!PBms*)z_pzF+<9-oSgER{{Uw29dQbL+aGE4CFJpWjkz)D ziHP&d4D*oZAvX=zfKqs|4yD&GpB7f*lCQv`$5SD5W6_wV8Lt9D#rP@6T!edMBk&>uCc|lTI-B}jo zT=C)ATs@Y_AFNwASn>FenrQjd)Pa`@a)*IUAYCXX*n#zEd$jZcFjZ$`wOo8lutK{9 z)*5P{3}9EBzur{@Tb`V@*~N_6 zSc9P3+X@+9X^%v%=b(e_7J zyqdQ4fY*06nA9LUOIHE=HjOicSACR_u$K;-u?`b|xAyNxo<48NV!%S4Ah6coW8lCr z!*9{5cfx*I&9fV)fpopjhgIjFzYT*wW0hY0^)&GI?wPi_(Ki))X79H)cbUfMte4Z) z_oIO;Yxazx%NoDJ8M0*B3SaIL=q}%(-|Y3(FUZ$lW;AV$k^UY=Up_R_{S}Ebk560W zGOfIz^N{e}F$|zUN<`w$gLkO7p+@qV*fiUrk}gKP0%+ zzIo>-li%_83V?A7Ft^_j;6k@qzatea^?}V;J3DpKRrup|_WvR^at{C8;6CU=GMb;Z?#bJ8i76%&6t^O5Ff)0m``Mqk&`Xv1Lu!|v z4J_wS9Egso;JQ8y`yu{Un$36p((bnBdvHpt)mx(#eJMdTCeV2LMPPH;Rb` zof=`H>!N>d0&l+}tBw zfX+`4+FCVN_Z-lTWRMw4(a0&eN;h z^W4if>RW}aG(}WxGxe-|k_}RP5LQC0cI?^P1}e%I0idd!&SNu_9{Y`fR(@ArD@(+M zL%pC50pd-kF`!Bf0oSj&@~({%a-8t=QafXM>U5o%PEb?rt(cc0CDs`4`rI6j|7w=a#&VCOP2Y?o#US=vv}y zXLag>mn%8=i0hLR@BS(?e%|So=Th{&M`*+5iYH#y_oX6^vpcWI+ZvE2?f>G%1`-^u zW@_xxwr%wYYyK&qzmXUho@i|MGZu@j>CLcC$O6peUuCi8xti@q{D`-QLGGEpyh5dv zGtYfBS65IQr^kA7y8FcybBwU#uj+)0W&fQC`8Pp@!j#Se;5gDsf>z4f;T-`w97FfrO2Fw$MK%k%RA zPBSRRr=cyfpO|dFwfB}enpYY2Inwm=UfthCZ*7?M->&<273ts6?4OGCzg6;YP)SC; zrHdN{Ww*)q^1jw-iz}`}3S=;Bz zc*kD7+*Jp&9sqJ@VX|m;l74ewuew#iEb^5t*nnEFbFyagygQg}L)vzzh)1J7sSIM} zn}_!8D+etkPd_|99a9!I*iw43w?1s(&Hd8T6QiZ4Mvl?ufSZNtn_%O}9FWZBxT%lb z+83lWb+d`lqtRblPt554voDPcg-F@|ef};7)gYNYo$qAuyl}Jm2WpH1* zBT6n64W663ZoqN}0i(Jh1?&y8B3y9QL_E}%!>rLkxKF+!mA}7fn}X7Lp3TJtQt%!n z*i+3v>12)zSlQVmoE7L>L)g-l0y?fGYF>e_{2{jrs}LYasLGMru#X?!g^e{>X6NKr zJT4lZ6cjjs4hf&4;T(h_u=)7W^Vp>|NTC9uk$- zozmT8=OaGS$DJDIeK|ER-zd!AD3N7aq_Ncg?>HN!ico3bVxx>USRe&Wl2!I^mXhpv z{b|uCZ=&50!4i#fuX`CF+ZJSrqHyRwRtC%H%IH}c^*ma1ES|945voWi73oRLDnC3+ z1g+YE(b4zWm<=1l;F_g)H<9{8$?ZWh2ek&*2R7*Y4}X{)SeDQ!;=D>U$@WW&%z84_ z;hjJSn%Y;l*Bu0;Rh}5_k)F z27A5_6n_|*g48kEB|At}a8QP@4^L;-ly4_@RlTLK%iRWO7_}maa6E3Z-nr@~ZmP#H zn?)NAH_0nu;73jHH^ISgRC1{Y`%DFp8DfC@`6e&1{ywCzI-vP0$;Bd z!!&x}{-u^Nl4`0p9JE34I$o$x%Z#bDMBLV-q3af|N^t@J2m$~IG>=Xv%fYhrRE_mD z1f}j{|6C$E_OX=X0Q#u!4nOCAjN|W>@uJZqHY=hpAPs-0d)b0*Me)->H^N}M8`>@b zt~^yC?F?e-@jhIvwUWQMkF9uZjF5ln8mWKL`5(Gc8^AAEI8b5|NCnm?ENMbb43+1$ zot&3|kv~#v6Fb0f@;0&O&`&2fzGSKg!S(3z_nP5GVq=X~NXIp$*qB)1+kub`G$K@C}{o?@SvLC|T1_U5|rC)>lAc9xkjv$V1(mA#(~K$COD_Lt`C zuh!iXR1CR>%Cpvn%zh8fLD`B=)j?an(s93D8?R4j+%D^q5XGmni2LgOq6^wG)n*J!1I-md*7zXyZY0@<|h%gefoXM0S}f1 zg4X8zWOlhDCb|Sfr6kS$pgV~x9q*S|ooT-02-)#N2PeiEfenv(f5ZB!k&D`^j+AK* zYboDqwM5Y6-0t@>2mB;+5FAH~^yaG1E%;!LRbWR856^53EDxYy6LkLljAqgP@34w$pcSk)_AJ1Lue(QYquJ5k?#Ugv}cfVD; z>Zz)y>Py@EZ9>aLy0lUmhL7pDg*b9Q|LZ0LwXkMhrY@J4+Q=-Lbar@%ko7B+na0_{vu zSqTE=@A&XMw5YXYFN2 zc3Fl^m&tD>8#cPa23#vIIaDdOcjQ?`^`iPbv#giiY2fCl5J-bUCU64FChcgHrB2GD zHZXeWso%*1X)2tzO>rD;_QFxg97HjP9FCYMO5RQ0=@}!U2lSPDk|$TcT#T^nVb1sa zrl*>pqc0aGsgz!8U(%M37|MO@DbhBGJed6U#>!qNw1mbWR+N_p?F&5J!6&hyGBq#x zY!)grRaG}U|1#1CqCRM&VRi2u-rZzmF(-smlvQk}8}Xb>h^?ItrI4P8BHVkKv6&6u;!nS@IT3JHNL zZgKPRKH&fK+xxF57xXF8YGzPr>?P?uKi&b{Nj{s1vM&yg;=NdDXfKEnM9~VmaO-nag39#j+twFCmlq`G%%yjK&yg3Gasyc&SG0hd9nH?I821-y)#9hszTJAtU?@ zYRq&9eo-nGIC&`~eY9;^=kLhhJINnKGDljq5%#L;oBRzlE#OXdoC*iULb}4W!D>xk zZu17LD|RB_sLzX?t&2(s3_nMJdg6^3I+Q}1#VG$3HG15L-k44bX`HVjIP)GSlf)zTozh`( z<7LSC*u}8`nM4$CK0+XDxcdtpKAE;OP{T?xZJBEczi~efH8Ymp$jr(eP~(V<=6v`KJ|%%&A!`Wod{?`2PKfWS6_IB7dszj zpG+~5XNFt1gf<)FJM&xSTLe-PT55**TG&$Dl(cLYyL)j0EuUE386%-#%jKy?-z}~^ zdf=UcOORh~rrM3*uZ23K^>Sv6g1u=X8J{iZC-}|Dpo&|8<+YcZuYwf|ZPH;!PrJG? zx|Mk7jDc3yX-qk;QK{AOG17gX{M4fcwzRdlx2Twm{8)8 zOLK*(7i4e9Y?&<4DcYy`>dfZc=qO`JC1zhXMb4v?XC|xYL7VLgH>JW;m{Hn_E*} z!7$rcNkz-&=(6y@VtI*4pINv3%D}Ut%k?88z2?tbgqG7ag;iIeuqmnQcI%!@FTd8( zNquN=8_|5BSu!FXgQvR;G$d@_l9_%K`B|Gw^R~I0Q!3Bx&7X)%(Y<%L5&GjiFTYt{ z4qh5TBv&2}4&7q*3Kg;cH|noPQVY&i44Z3f(nc!I_EF~uSV`y_*g4;UP#8GsH&P(6 z?MV0JOla1{6RyPYD`vtx58L0SOX?kT?cX1`fcPg3F^Lumh zO;=N0W`qu(TQqyM(9d}B>+YnrdPkS8Off@Y>}#Qbf@>CxljBaXTpVW*Z0WxY?GPh{@i&RGjVgLs%hG|vG9gvaZvTkOSWEx&A8 z(KVBKzO2%7G(S=nBU{2R+Fvj>bXz)apIm%$3^nP|Sk$uNmj92V5vxr)q(7=(8@%^E zqTzSMV#@{&IW`Y1dqcwb|2%?75(%$Xm)@=SEVOb&Ni7|yVxBLw-XRC45Tg}gfswih zE(HgUzyLNlibvsOn7+Co7WrjyTgXY?KL6Up^vdhOaOi>gQCiXBR6xDXx55{If4acj zzPl<3S&Xg!>FlC&8nk9(Uto(c{KZA`up@2sx3hJqR!xU2z8H}pa4YF#Zs-fV2-tHm zPuBCEa+0w4?MH=-ldf)M17LL-BBIO=5FDRSUUbp8qSQl%PSlaI6M5@YC&BU$@${xj zR@G*H^D@0y)8US(BLoToOEEEIc8VM~&-?0wVYKZhjIm+tm@pLMNA<=TDNkyy@Wj}r z7PzHj(<`^~GX)cieNHjX2ed{F8v4^_8fACUZ2$B#4eWr(MlW``q~>R|I!PNL=?P*o z@ge$E(|+i-ek~djJ)z>9h5c?3{vIN&gwWg$dYGzZ_Hq_u>s;orA;qli&BvHER`*kMUA%ac$@8CcGe`t`W(@1zV5Y)SaTS z0isF_-5)pxCIwc0tkaVm7{ zJwx+yFRj~NyXYF;i};5U$tb*_hS1mdK6!7{fGH+Vz%$WXr1^^om`Ri+;KZ|E=k9&| z1P~V2NKRYA4+K9iqtG+@p7fn37V`e+_XRv>qjZu6=G}>K4BTJqTg1oLo{*_j zP;+iu`C@|6*`Dw~*|U1q%<9=g2{IjC`uJR_m+Z2LKN0NxD*d{@#3#048Bb{5LB%Zm z1zxtb+^ti!@|f)4&`11lH9ZoiNhQ==tNl@_pQxB1o~*)Aa0G{?bAU*SmFa-<69V?G zYmFS8S?hbzx{@BmRN02^wb*?*9sWwIc7lNL4|CY{+Hvc~vT$r8ElL&~S`>S9i}kXC z&iBh*V`P}d_#HU;uJy73N7|e$lt*741*jw^>zJXIMcbjhnW=F_Tw-}Fx0IInmm8HSbwr$F{RCZN?;iMZY4f&38}4OP%cZF@V*kY zlsXfhrXc(b<{dO!j(D2A-K$>*CzP6OSboHEk$lVCFa4b;4!o2#^He{65w1l!g2 z?(-vEpw=5IrE$^MFV*{fk4qVKhMm{TFlcfJ8ZFla_E4fbxY%g zdR7%;ns|xzmG>V1rxpp~!BGk)foDBiwI`T$IvU&Q3W<4dn%}acN@%g-yZC;D1O;~3 z2fQW2Hs^8BvOVjK0R{kcv&VZG2=x8YxUWH=Q|r~%fj|c@{MRpBFrPy2(TR+ndz?6S zEl4+Z>Xf8?x<|8f*>l$(@2260s*QKlhm2*gMTphK$f~6ubj*J_xMQu!qzK#|v?#wW zL>ms@h0<+&YF%S*_xBwPQ9jfJj(zH|mA=1X$$y=J`#s(8x7QgmUU}VaE1nzmM4uNr zCeZyA`rS$|S5{iYz!O~FAlVZOJrN^Av0`0F5!;9!ifog>-gs-_X3T*-{OX5ZSjD(} zb?RAC^K9XR$+n+KW8^Nqhb&3_Ofbm)FB|pywQb{2-TNmVT*8#2zlyRX9-hPL)U%>2Klo@E~a?Nw8b{B6ha)tlhqv;xs-EP z;1$KM;$+2gvU&}H+P;{8mhtGq53g-Bib#zjd;pLK1aNwb0>slSd_?5!Nurz_N_!@$ z_pA#o0!kdVBV|^Pa>rU+OVu$IWZ3B*M?^81C2&atTiPI;EKIK+Z@K!BfIuv@B8bKG z2|w)OD_F0`@NDMG;bAVUAsE6XhtG_xHqnM#jhZB=1a8qiiDGe|kAglNiu*bW5YY5W z|0%Vh#d#i7OyM8_~S7$P@x}g)`!tmg`$A6-#ra zajyYJq-1$}!~`a+pGxOgFgeCnZzdYV4FbIS#o*gXl{-ZddM@Sm%3N)bgz!S`@v6G) z90=7HFH+IgR!bjV;sZnx?>h7d_OQH&OCm8tR5@IOZu(m5QA3nC?eTy=Zvz3)w&sj$ z3GbY$uIe?7PhVBktm*V9t6|EI&Od3`U@YrmFFBwrweFUMudGLx5Llj|A5Iq5Z+ zt<}lzuP8)Oc8?#qyD)A>sYPusF2{#(WSc{3Yr<4jhk7m*!l%hQ)XS5;d#r3Qd7 z4jUUeSQ`QpWgc159$~vQ&i>|X^F86Uk=3f|I=mP=mpefm-V!<6zC$DG<}iN(31tLC zKBgwK?`k*#%mNsILde7bH8wO5IwU|EIJ}DLs~dJ^#VVw1wuRAMQ6*GnDH&)~zJ0BV z4A6*D*)!&#?OMtWlr)h)yjWREIz7D5YNl%{Q$vK?NL~UAhY+BC@=fs8kHg4#@o0*3 zz18}W$onjb(%wejwdj)&i{bl;h$RgEFzQ4#Lt^Mdn5y zXjnP28QB0<9WspJoTQs{Pu zSRv!D0=kl=VJwu3DAc+BCaQGIXL_p|Vk-8yBQdt>HUxn_iCIJe@ zfXKJOpoNcqYU0YQhL@`UBH}?052bVVLP|Bbg77xW-nl$+{c4&hHuphNp!0Y)#sf3( zJ3{AX^l3rjj=cNjWNt|skC5y)xQ*I5i)&lA`@E}3=*;ml{o!&6i!EabmnH!6PeLa+ z=9t1SchM+C<*a&uhbXpKbt{&^im(>d~qU{H905;ja+9kdV?&EoGd zD2AH_FTv!oO6P|3<=$778hq$UdbEQgTN1vn^Mewvk&CwYUbo^E?ktYaB^ zmSlz)&RwVPy64lv7xL8@nFb!c$xvYdL*`a_sc5;X`dN~g&&;~ioxA_wj7iNC8;-@TMX2S)%nvxfJVW~B-3ua$$6)N6o1lUx4*>Lb`>Fb- zBtn2WpGYfNIYgfw>!g)9m-Z@In@C?nFH3kK_c{uj;H@Hap9M4@=#%IMvkn6V_ySA!ZEr^hBJ*dZ3gCvl0mq}}s&{g1Rbtj(!324gd5kG=oXlz@=Y+x8l<>WB8b zd&GmD?|Im?uNaVipVh+tzl{0Ab-n_<{8gp%KA@&{;oFH-_X?{}D?s0AZq)Y$1pjlf z|9LvQ_=6gw8)He!-5-LyI#><&$lmAIaiRT7U^E@^(h+cqi;*8fR&?|XTKolc>-daH zAg$Pe^B(9<pwf$~|+s56szP&byJ30qzZvsKJJ)F!u_dBks>Gw%1IkSpq(Bhtove zywfg_w+gg=`_{VrV2kOR(Ow`S9W-^-f`(rpPZlI2uWsu;tV|rp4H0wWMDL~>!Bp+g zKbqLl&_kod80kEh9jeCQ;IMh?457JcR`dPHj$OD%QUQfxf?$-nah_jaPUi#n6+JM5 zr9}-tr2$U-KzC`=jjP0J^<~#C?0dHIr$?S~qE|3$BUbomsM^u@gOGrFk$voWVm-Du_1jHaCoWRwLky<7ez zll>Hx{@`~07qyHXnIr^34)62s$~Mshh?RVD1A^UX3{*qcyFW*hKe!nXjt&o+xB?+9 zkY$aWJe6&M;vW2K*!pKrEy(aZwVA)MY@iZ`lS#!cq|qy9a$`2WKTri{RueD0y!B9- zOtv$XE=!EPI0XbuTT(`f0e}^grfc@OC0akc_j!q%E47LN<((sut_<86r(z+mKS$bs zl}Q0tTM>EytKmeF-3d5P6dr_>@8L!UEMcNKP*? zQp6(ATqO-Yrm)bLg#cd^#)^Y^QRSnjOhXMQKMA!zyEYNbB}op0bz z9nIhU%EdSNmj^Y77)T)9CypC0b!)f2On_i=#iJFMmeJKTYNTo3iaz zW7A+wuy3{*0-C+o_ozG3|FkB+b z^kC+C=$E3P4f|g*dV4pY@Gp6vT1=eDKcYHgT;1#oWp}|t#M4XEJj}d+?Zrwnat%IO zdRMQ!W=q3|n}$FBowt^#1pbki>0AR);F^Jx+GSs$TsT{l7cT=Bfkq&%He{lhL&LdX zccuthfXVIq_JANjtGVXkm*%){hfWVY0D@TI-8#A5Ac+;~VF5G^dVB1^7$wa$4UC8n zI>`w+h{pX64_??@*@t2}vE~tsAaJV%bpBaZk#hKy^BW=Bk1QUJ(#T0qqz8dfOp=m~ zCq&XqO9bCVztkSSts6B}!KmMXjo#NVhY^&`!=u9|Gd+R=x?VPK{+Mv*~miFhfba)RkTB^aiTf z$`9YtR$4|5ch*^cxK8AAbuxA%ToF1L?-bE-rUZ3!L0%hflG5H2pwxtNBHTl2s6MLQ5IV0dfaIE3fR-_cBGg$aCWQPlm7VN;?ZZFyGw_YIz~dzlh%&tAJdO( z@9CVg7e`H2qboy4mvv~@hW%oDS!}Dr2hGnLk64l}ZVX8axDq zQ?>cNIT_8BK4v!J>wr0UT4kQ7j3IN~ov9J*t*FIHYZ(XZ%brZZw~|QYvOprm;vIr zizmui*{p~J4m1o^WN?kGkRIrzp}JvFIT1$9E4iM$edY5E{Kut%(-BR=c@o766Fu7= zzXp8~zpwA8aW&O@eTbYcYF zDTx8unI2sRvVj59|C~D~cOO@tei_OR1$cPpREO@A4My&BBcaAe-T}?@$$k(jTbGRY zLOTkUo6qj~7PS6br>H@idYy`1d14ap*|Ys2OE-38D41tr~@(YY$h!2*dN$BxmfV0|2xrYp4vMfMjG-LMFO} zIh4hm0HFI4&{J@P1N4qBcO~-(-4?tuuI9DUb9in)h|d11YzNYVQttFwAoG#epk?cg wk8S|jZ32!n#kKBS{+}4Z`marM%_~5SeHzn&MGiP1h=YzFakMQz?0w^Z0n2T7

+Ah`Khx7u+sl1~ z8w3LFRljjf9|YQN0s?Jw-Mtg|WQ68f0s<*8)vsN-sw?v*DkL|hjEcWuA$o<;Y*R;Ey-kg?fe|!Ax zsrI)&UfJgSlskm`=~IbVF2;ve8k;(A>R((;P3mWDdq3ybMntqNc@JJ9oqIpSVVo8A zpJ@K53miqyaARicNHd4I`9Yw|*qa5wFImq6xPY&04@!VQ55?0gw}C*n4s-E5JrqiYgh}vg*x&&2;`S$ zY53d!K`biHR<=FadsH4tO{y=-S~#S1MK`$hrL;<`{Io=Wp>?p}9N%;Pfo;m2M zYgNTL##yb0HqP`HN7Z-L_dwGIt*7YC4*7%&aqM~2yzgQabwkSg`E8$p`JyJNsHIi~ zNL+9egk%vTVcKCQyfWaBFA|XnzFv2GUYI?IZO!EfZG74VeNo(|Y@S!W*f`b(p7hey{+uN#X0HWmrCUTZ z+_rogwH)+Od01O+sLPgmUxDGf+Tx7vv?bv<%NbScinG?y15~rZ+p0yln6%rB2O+`^ zx@e-j$5M-&C?yX+mG%0qm?X@{svM;l)>;_8%c?5oW|i|on`cb`7wm%yXMF-Hz%cM% z>%?JZ%{l`5*z=99y|w8gt#cVPA4(ZAXk)u1O`hmN(@^*UbpQ1o?jBVESajlv{z})o zi?0rs^=~W>q~h`TS89_NY3bloZO~1>qT62vUE4&;-K+96exM~s3=!Sv9ksyA(5;o1 zn$MAW(uO>ZNxNWz-XqUM@rrFDo{{~4)IEHweu7DMZp1Q z;x=aPlG{XfLlH1Zvnmgyesxc-d2G%hI-_3Lzq-KLODoBDC?hrXxap4_NPNJ-$*feT zPS2}Tqja1=a#h}0zRufkDW&&!7BZ>WiM%?ka(W0Z&rJ5hh12tcUC7V=`$Lr2GsWbW ztp`Bw^5ex!AcSRos|4b1`UNFE8)m0~mnNdxy_wjeRIRCH{3^hut=@?@`(_>Oiqc}k zNf=|Od;W9HE@25rgU{Ymp=is^=Ps1lS&ikclsS@J=V^soj6#oziEOZ3>J5trb$m&U zSZ=P2RoHV4T&z1PW-To?C zA;u69X$Sla;;57ZMu5eWv1maYHP0}Cp z@xyM4@f#&92PyB4Ft9jVn{{PjG}gDd!2ZC};0)tFCGUrziA;e9y9(Pn0f9sZ>w_QB zgt7dl@o?eRmtI9(=?~5*|Mu-uriHbCo0dh_{}6- zP<+VWIN7&Pj*<4W8{BPk#PAB}AvE)h$>$+7<4EfLM=lGp()gm=h%dEbX(RQ$ooYT5 zcto22lcC;WsdA;kGG{jLJRHpFd)7B$=WP7baQhqFF(<5V4vfVbsQ(y|_K{-Nv5YSP z^k$?kl!ww-7-4GXE4{&@EBmxyJ0?=*FLzHE*z>g&F^vJeSs6;jJ1-<~*FxJ3rjAG? z2?&lJ0UVJe9~b!GLBeo)SJj{pYPr3ZxR!Trh{2|ADp$TyqpmPVg*PjjMyEqFhI{y) z3&6dO_ZthF>k9OvW@lX}?~y1i*yr|%%=sfc;A?B{``L=`#m=Av%%AIdxdnX?XPQ(k zE|s?F;;PVjQBGj=-Na#0#W!dw-u|_KS6hM{q(4tMaDQLDM7h$sU&$&RmmR_S@}5_e zvMl&iBY#&XWo_;{c${Mo$Z_@JZj@Xmx*$7G&n~hI8u7CtbmI&!pjv=LeNdr91IZbK zV=$N)Jo6)OUZ?Gf3*Pjv2@kMp+4iPg&d(=Dw4us~J&A z8UP#!ZMv2RGE;=(U(wIl`@a6(z06&pRcuH=`f`vK1m1&5$>??Cx&aMsgul$Sy<&W1 znM)^Mt)t_oJm12Ie6Z}AsVaWtpmMi@pHAek+!=SOJ|Ipd?R5V<^y5HfT^Ow(Bl|Wk zgSda1rvlx!MlgQmA3m4CEX4cf^lcsY%9#smbJdq2;=R0z-3GR6FMamLqtDpQKtCJU zS8=KWN}5vK&yqlJVpbLd8$_2I`YH~cl!jwcf4s+fj$A1j?!WgI^(Lw9Uu_#WEJs{s7UGtgT6!r7(PipvL~qF zx-JhN>7xk+nbuNIRb_t|%wv;1hOfH8t zDl?9g70;d($3rv}Xz{06y*x)=y<%^_EUQ=W@QAJEt@K+>Yb(>V_rjclp}|~F5_?F5 zl*d(e&9HG;uL&v)^4-P7^Y&)hWkZe9^*K*c6vUs^@P> zX@~HOCaEfkO4N{hFZt3$Vw9f}dVTe()kkD6+&B09`SYjJr)P@WoczzGSv@P!_|EqKC67stIMk@~E zw-E&KyK<*3SrW%;zKfx3X$Fnge>HGsu=1R#bo*vjgx}c7AVV;$McflPFX(J8O0Iv$Fq^oPShxMYiJw@K(mi zV_nzCw|fX0{kW&6N5^`t&qrYl_!wcO7xB0jO;gHw8+*tj*%g4OOB7^wix1`_sGH>{? zixuB|d4}Votms2nbhy!<6-#>ACrjG;!zqDVU;;-i|2TfJ5gbMFnQ|De_72)uIg*Ol zm@@9@>w9@H#2Gbh9#nXn#{J8dHjk~Hf(^CT%TJ~Ra$EbbfyzVEC}27M%?P&Go_{xR z8<%VEz3Ou`q@mSy}#sRFejR-AQS^IQo37)|qgb z{rG^+3u6STd#_0>uT537K0?j-BCt z`cBD?U|lmOqPKPNVA?GVy%xg4>sJ?)B$MW!Vzw&avE_9TF?P51fw-bxU z4OB=%HoaAQ9f$!FJ+!f|GWH$U*>X)jiSCb>STLu+favSXny&cx2*3?4^MZ|^-14-Z z8do#dgx`Ztq0K@{6zO&!T!{X6HH+kJU7Ko+OpC>0a}v4ooti~ddB1`VlUC`CdU{Sd zL}>_9DJpWa;T zHueuyANW!5>(iVye-eT{(%I>?W@=>ia(P;RW@A~<-@{O&crxA5rDp9JQWdHBtgWOd zWB0s8M|@&@7Zj0S*I^L;)-nHJPD_GZ)0JIjSy0&xEyGWSeMn$!JgtbYK{jV zJK;Lfihc9ubeyb1%rVzy=^*JLG*E-{%???0yjGX-WjJQ$=f~Q0z9Ike>_FNFU=bgc zUjCBceHTB0;?CXGAqN=d6c*lOP@GA%vxm>!X<$v@5G886V5wQz*_uQ+!Kd`OkdnOp z9>5ai+^7K_?*Y^SvuPx5;?SFiJsE!S)qUUf)djNMd#m?P$n9#vk`6sNV$b&;mboSq z$MC#d0>BQRY65@TnJIaX_m;ScfW`es%?Wb(_u21u@J6$iCPZ!R+Fz;4Sk*qR?R&Uw z2RCA3$(v3k`F9-xa3kkQBG>GrIqn{y@EUu5uC*Y6$E7lHkDz?Xec#nDpF0KyI2R0H z=^cjIFQqjhuMv1RvEz^TU%e4~Pxo!iBCsPCY~o}8rOQB9yHW2FKBRP$!xnIgIDGE< zR{iRi%M_NizxC@*XNCf@Qf}3BMA#jYbFy%GRFyt|w#y*UERSEpQrx65yb90ICs zEEgP2{2C5A^3~ptmlb&J+4^zW+hNMZr8B^Y)tGqNd!)NH<8n-HOJip5dGc!L5!->> zmLfYf;#mgU_AqM@8%FM{Bss<8?iIB5gdk@ZYol8XiHI*tY)_9$(s-TRu+NJNJVtkD z%hT~TwGpOVMwH#d8_wsbT&IzYuA}@-wG{yuE zc|u@th{M~R?*z83M-09^nEWQmyYe1fQ%0!S`(DQS;^(9w%9V6BHcjLU57Ys(ZVuCG z!Y1dLuHL@jeQedTDdg);pkW!BfOZmhPh;6n%Q$1VoPD`NT}I*P4z#Zd3Le8h%kIm{ z@tcj+4x$Kf@>J)%#SJP1Z{ZP_1gMk~<{&o@c`Mcr7IT)}g6jFjhux!dC=sggct+*nq& z!5PI|?{6O@>KjsD##-N%YhcdXM=n5}w33;cN9A<{UwQm-la((#hgsu$TY@MG=Vl=< zQYX(y>f>iaf!ofiQdSNQcV-uNbcin|)a5SHHIGKIjqwOEIp{!qjwP4bn`NR;Qf!Q} zAa2lT9PfXyYrDCT;?4(v+I-uy3bV6$EtK^QSGZ<^1M1)eaXgpL;wbZqU3T zgY=B+^Pn0J&s5|mt}aLrb%gQ3}3{RV}Q%AuH%=Gyk_PUAnUWY zr*5`<#Q~Pe4Ig?LbpV1C?hG9jHW_9q-acb8G?X9B2IA@FU{~csB(hmFNcdY8q7`V6PwKS?D>n3S(t{sEoF5Mdn zv>A$g$cyA6_E1#Z*uhlzOy2zUE`!I|x$@hn2}7xkLrS`cqf(lm=Qa0jCO@xzpgTx%_!X!y_h z0+rLFQvSK451NK_AwJLr%!gWBs;w3@U2}(Zbminfr{&*^uD*KvYcOS z2{d{9f6$M5OG4=5CvjD;ZvG?1DH62aN(;Ii;%(Md!(?WA?(~T9KuWng0i$`T?9eJ*oK! zD*WBmnxc|3sMOqRG;KNal{-2*6%{v={gp(u6_#AutNB^2$d;f5gMpBtu!xeziG0Us zQMlMfA4sqHH?6cE*nyxIUfI>(Qsl>SIef|G^`0Z=Z+r+B1cwShhO@1|=<2p78Qv=B zK{_D}^+Vx}r?U+n<;l=%s2DpEAy*8SXx3#|;KHTJWJPEmu~l>JH3*GK-q&R)5gI_F zl4IEt#>dyzKL;i$L|jSC63D@VjT=7)3rlURxuYi4qfV0n1U#Kcgul^FIDJ4=XAs!Y z8E|YLV0}O9=C?L7%6>XO8Xj}Cro*9_=&hp08i=#oYywA%DA^t|deg|4lOT-LHYyNr zpRPE~DY-dS<|%pmpv2}3&(r#MyA+%On}nHv6q5h+aEA?$t&8x)3o zjY8mx`KsEwhA}F@`kdrHtZ%5E86a!hA#Zb-aaDD@I6s_7?80w zDdW7V@6}Q_HGuFZCHCv>A`ibczLWfdWDP~neoFHE5r#`iOr zZ~iXrDEuDS?qn3(Eu#~y&?}lQMWX4dXErG>v|-Fqkp!NPqkmBLgE{Bz?Z7g;bH){M zIU15{mh7-`Sqyc(KDZ_gq~#9(lO?)&xQt77tpM|ZTfpdDZsd}-~1;G1SHMt&`GA{}_sqBgXV3q;qMB#0tuV^~X% zPQ>MUygpZq;DjYmbQOpq)Kz`T>Lz$_r%&kbHxCZL>f`03y)E_FvU(Bg?)n}fwz#!h zGBL1=S-Bvjm}9;h6zjVZtj@}M-Gpv&IJNZgEZN!n4C`ERld?+9RqX=E&wtkjy@HOn z?$+j)e4NcAy$f>HWlQ*ygl{m&*0XS=NCw;myqS?Znb?I^7)P^=$P!+-F;7K`ca`BC z(%>56bg#FqkdlcDm?+g>&mneR{CMcR5<8hd@5rS(}wr)slKt~aFeOo z*q=5pKsZE<>S;mR(uYncP&QNNi`ewOvigC|LD2}T-^K}rJ_t6cickb(n( z;%#{EMLZJrnEQXaoEV1chO^;c53>8{_RvPf4ld4#xTp-^;#x z)(c!G{vm6yD*hmU@ZV;(%}RfH-sBajuJF4$F4|BhF|^LOLW{{=`S{$0y2->aGG%|~SHIdnGVY#DlEO=`4ek=A58 z)(|Ok@8i#h6+Chz;MEuHbMn7+8J!6JoZE6#zv4uIClEy0s^>?;74q&y?29_Bjp;AG ztBFDpv=T08pVmn{B-_*52UIPO{^{v6jJbON#scl53NLnie7v^oACle+p;Tgsk4-+%R1<=qZj&N@ro)U4!Qq}I!`sb`J+a*dR6v78OI zTG)y#kXCE`v&PUD0^BLcaxz&I=P2iiX`FqF z>?%7v`R4#?f{fGP#r`5&O)Ez-MnvZJFS`$4yl=UfcV%NH`(+{V!?$!6xJ3c;TRH$l zlM~+(B;a|Ue(Raen26m5g!aQ6u^M zHBkVFrq7dv^{Wbj0Q}EI3lPi1WB%%FpLJj}l?1;dFb6J8<3a}2f3tQLXlym$-KR16 zy?HQj+V!x3e<7@Wm&RUwC?K*)&OiW!;F5j)!!0={1u1ffHbb@Pl3ErY*S;iynRZ`dO+6;h54xr z;5pXzVMH`-etG(%%>7UA#$bVK{l?PY_vlvhMgQ}wz5jilH07~h_85FE+(yVd!bZp^ z!zLDW-DcwGgHe{&q5=lRi4NbOVw_EqpVQ!kcl%Vg(t@*iHZ1FW>%3Vx&=dMYK>(u} zp$&?3*F(n=jGmP^MGuDFy}?-EEhpI9f+I*PuVTGdVJG?plZ2V+=R}q=$`@V*2RQl1 ziia&#^&o*B%^ymX$jsR0Kz1o1g)mqsiHn-4RXIgsF|qaUvuX`SPchfrRoNv-|FM*^ zS=^W^@2vQSl;M#H&s+-7Z~WbX1-5Phlg$Cl{uT_3Q&%RCZOT{wJ`|4L2*beE7HYkS zPI)%5$4*xAQ9r5jKGPs+Vn488)5?De1u4a2b?Uk`hbpm~Q5HM~c8b-Louo2o6XJlfd}FvnJ?N~GiH=T zVQgQ0VTOJ1fLcA!%99S0B%|E z4WAg!4v@8J*iW{RA@@p3O3HYBu`E`jT74tYJ;vm6_(L^7x>w@-Efi{NLBG}ec1WZ) zv4>9r)-FCVm7O00(E8p3T#DVtVXu@@%?1T>{yD*y@delm%M!pQQ(c)duH#yE_wK!* zcV;w=k)vj%fbWzVQ8i{oHqV|PlD@ig-?5j~X?jqBt#Kc@O32o{Pw5k&ar@VKAi{O( zi~C(S)yR}#M%fq%Y?O26xuivf9>D)Q+wVz5RXyM4I|9)CqtP4ko81{CgfP3W-iEZn zi1J^{@W&Wt8@V6Z<)L~Z0o$nyRbrt!;JQ4s~EE26f@ zlw$hdV$-NHfAFBzLpcjl5Jxp(R0%*@(Tz_?pv_gy$M|z5{O8W^n+e&PLTOcIJK8xq zMgue&B{5T@OsdLjp5oB>0_*y48U@pUg=X+t5;iUE=bH z{w*!BMF zY5@Ds7vt|VJ?uMgTX)cROmN;S*y4T-myBay!PtBr!1_z?^s4;z81PaQ+rzuJY8dOM zH{#YE2u4xmXot_3S`GnjD zC_X`jbH5(Yz807x5})^O^PRb4TILv+_~64~k8=f0zpkYw3KuOkXDl_AB2J)d&I^@oUnit1aPDPPVAzb(J zGXN+q4MQSD$1s3VquC&7w!#gv2ILMu;rixm1F**_7E}>-eUs_G$*g^e)_Py@V3b=x zXh*jVqKiXAa70+JF3;R4jTp4ej)E$nnHp2gHr<@iGoFC5``ZR!KQWm5>9sSjdMDi> z!fPLcF`m1Hd}j(5fiWV!5Bro?EjbWg->?l7H!e`t>&vGxlHc4B#$)~!2#T(`m9vcr z2IZ4D_#qK1|F4hsKVZD=_aQp{w9?XUr|l2nS8BHF833GU;|2it7;88g5iOeRRXbiWF4vIQYR@c-Lj>FVy`YdyNMVwsac+ zfmQiL`3!t+Gj0mA8q~&Kg-U5?bSF3y#SrX)-mz@NM%K_?kVXhQVJFDvspRD?5_GCc zFuxO&(PzEb9)Z3E(9EEh05q8P6Y}GYH{{iHYB$7^`oGwcvru<8@`F>3DP8NdQKMJm;etH-+V0ZUKhd zl)pUUbsLyCN?pBPB+p2ouAde&O}{55zp>m&!%yc$@qvx{K1-?UwQSA2F$wb`2>)qw zDh9}KMcNE_hJdj`JI2lzcVL@OKo8+I#*fd(TDUwLl1vpS6%a@o$uIRn4ptQ2SNMo> zGt61_5KFj##B9;ew}^bA+4^QP$$pFE+6n?LF9$VcfWufw#SWsW>|xP3tyAj%`j z>kJPuvxa4}f-UYh#1TxyC3D!on4OfnyL%847`D>Qz`PZh*)gdgzS*)Jq}0GKq=aS0 z`mrv_lq`L7uJW22>;l!*kv>QTY@Nw>W*aD((-Tdq2$!b+Ec zEl@iKQKnJ?_l;Pee5jnWOqBqA)C>vLtk^jngd)E7vqJl-;Z@D2O=lp#v%T@UZrT0w@7J6 V*7;Z7!oZZ&@|@~&D2{2zCVi0c3V literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/huggingface/launch_on_modal.py b/end2end/liger_examples/huggingface/launch_on_modal.py new file mode 100644 index 0000000..1171ea4 --- /dev/null +++ b/end2end/liger_examples/huggingface/launch_on_modal.py @@ -0,0 +1,69 @@ +""" +launch_on_modal.py + +This tool is designed to launch scripts using Modal. + +It sets up the necessary environment, including GPU resources and python dependencies, +and executes the specified training script remotely. + +### Setup and Usage +```bash +pip install modal +modal setup # authenticate with Modal +export HF_TOKEN="your_huggingface_token" # if using a gated model such as llama3 +modal run launch_on_modal.py --script "run_qwen2_vl.sh" +``` + +### Caveats +This tool is intended as an easy on-ramp to using Liger-Kernel for fine-tuning LLMs and +VLMs - it is a reproducible way to run benchmarks and example scripts. However, it is not +the best way to develop a model on Modal, as it re-downloads the model and dataset each +time it is run. For iterative development, consider using `modal.Volume` to cache the +model and dataset between runs. +""" + +import os + +import modal + +from modal import gpu + +TWO_HOURS = 2 * 60 * 60 +SIXTEEN_GB = 16 * 1024 + +app = modal.App("liger-example") + +image = modal.Image.debian_slim().pip_install_from_requirements("requirements.txt").copy_local_dir(".", "/root") + +if "HF_TOKEN" not in os.environ: + print("HF_TOKEN not found in environment variables, using an empty token.") +hf_token_secret = modal.Secret.from_dict({"HF_TOKEN": os.environ.get("HF_TOKEN", "")}) + + +@app.function( + gpu=gpu.A100(count=4, size="80GB"), + image=image, + timeout=TWO_HOURS, + memory=SIXTEEN_GB, + secrets=[hf_token_secret], +) +def launch_script(script: str): + import subprocess + + script_path = f"/root/{script}" + os.chmod(script_path, 0o755) # make script executable + + print(f"Running script: {script_path}") + subprocess.run([script_path], check=True, cwd="/root", env=os.environ.copy()) + + +@app.local_entrypoint() +def main(script: str): + """ + Launch a script remotely on modal. + ```bash + export HF_TOKEN="your_huggingface_token" # if using a gated model such as llama3 + modal run --detach launch_on_modal.py --script "run_qwen2_vl.sh" + ``` + """ + launch_script.remote(script=script) diff --git a/end2end/liger_examples/huggingface/requirements.txt b/end2end/liger_examples/huggingface/requirements.txt new file mode 100644 index 0000000..d6d10e9 --- /dev/null +++ b/end2end/liger_examples/huggingface/requirements.txt @@ -0,0 +1,6 @@ +transformers==4.45.2 +trl +liger-kernel +triton +torch +torchvision \ No newline at end of file diff --git a/end2end/liger_examples/huggingface/run_benchmarks.sh b/end2end/liger_examples/huggingface/run_benchmarks.sh new file mode 100755 index 0000000..cf4234a --- /dev/null +++ b/end2end/liger_examples/huggingface/run_benchmarks.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +## Benchmarking Script +## Runs the training script with different configurations and logs the results + +MODEL_TYPE="mistral" +MODEL_PATH="mistralai/Mistral-7B-v0.1" +USE_LIGER_VALUES=("True" "False") +BATCH_SIZE_VALUES=(64 128 192) +NUM_REP=5 +MAX_STEPS=20 +DATASET_PATH="tatsu-lab/alpaca" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +mkdir -p "${SCRIPT_DIR}/results" + +for USE_LIGER in "${USE_LIGER_VALUES[@]}"; do + for BATCH_SIZE in "${BATCH_SIZE_VALUES[@]}"; do + echo "Running with use_liger=$USE_LIGER and batch_size=$BATCH_SIZE" + + for ((i=1; i<=NUM_REP; i++)); do + + LOG_FILE="${SCRIPT_DIR}/results/${MODEL_TYPE}_use_liger_${USE_LIGER}_batch_size_${BATCH_SIZE}_rep_${i}.log" + + torchrun --nnodes=1 --nproc-per-node=4 training.py \ + --bf16 \ + --num_train_epochs 1 \ + --max_steps $MAX_STEPS \ + --model_name $MODEL_PATH \ + --dataset $DATASET_PATH \ + --per_device_train_batch_size $BATCH_SIZE \ + --per_device_eval_batch_size 16 \ + --eval_strategy "no" \ + --save_strategy "no" \ + --learning_rate 6e-6 \ + --weight_decay 0.05 \ + --warmup_ratio 0.1 \ + --lr_scheduler_type "cosine" \ + --logging_steps 1 \ + --include_num_input_tokens_seen \ + --report_to none \ + --fsdp "full_shard auto_wrap" \ + --fsdp_config config/fsdp_config.json \ + --seed 42 \ + --use_liger $USE_LIGER \ + --output_dir model_output_dir \ + > $LOG_FILE + + sleep 5 + done + done +done \ No newline at end of file diff --git a/end2end/liger_examples/huggingface/run_gemma.sh b/end2end/liger_examples/huggingface/run_gemma.sh new file mode 100644 index 0000000..c882f5e --- /dev/null +++ b/end2end/liger_examples/huggingface/run_gemma.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +torchrun --nnodes=1 --nproc-per-node=4 training.py \ + --model_name "google/gemma-7b-it" \ + --bf16 \ + --max_steps 20 \ + --per_device_train_batch_size 24 \ + --per_device_eval_batch_size 1 \ + --eval_strategy "no" \ + --save_strategy "no" \ + --learning_rate 6e-6 \ + --weight_decay 0.05 \ + --warmup_ratio 0.1 \ + --lr_scheduler_type "cosine" \ + --logging_steps 1 \ + --include_num_input_tokens_seen \ + --report_to none \ + --fsdp "full_shard auto_wrap" \ + --fsdp_config config/fsdp_config.json \ + --seed 42 \ + --use_liger True \ + --output_dir alpaca_finetuning diff --git a/end2end/liger_examples/huggingface/run_llama.sh b/end2end/liger_examples/huggingface/run_llama.sh new file mode 100644 index 0000000..b6a1fc7 --- /dev/null +++ b/end2end/liger_examples/huggingface/run_llama.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +torchrun --nnodes=1 --nproc-per-node=4 training.py \ + --bf16 \ + --num_train_epochs 1 \ + --per_device_train_batch_size 64 \ + --per_device_eval_batch_size 64 \ + --eval_strategy "no" \ + --save_strategy "no" \ + --learning_rate 6e-6 \ + --weight_decay 0.05 \ + --warmup_ratio 0.1 \ + --lr_scheduler_type "cosine" \ + --logging_steps 1 \ + --include_num_input_tokens_seen \ + --report_to none \ + --fsdp "full_shard auto_wrap" \ + --fsdp_config config/fsdp_config.json \ + --seed 42 \ + --use_liger True \ + --output_dir alpaca_finetuning diff --git a/end2end/liger_examples/huggingface/run_qwen.sh b/end2end/liger_examples/huggingface/run_qwen.sh new file mode 100644 index 0000000..54a157f --- /dev/null +++ b/end2end/liger_examples/huggingface/run_qwen.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +torchrun --nnodes=1 --nproc-per-node=4 training.py \ + --model_name "Qwen/Qwen2-7B" \ + --bf16 \ + --num_train_epochs 1 \ + --per_device_train_batch_size 48 \ + --per_device_eval_batch_size 64 \ + --eval_strategy "no" \ + --save_strategy "no" \ + --learning_rate 6e-6 \ + --weight_decay 0.05 \ + --warmup_ratio 0.1 \ + --lr_scheduler_type "cosine" \ + --logging_steps 1 \ + --include_num_input_tokens_seen \ + --report_to none \ + --fsdp "full_shard auto_wrap" \ + --fsdp_config config/fsdp_config.json \ + --seed 42 \ + --use_liger True \ + --output_dir alpaca_finetuning diff --git a/end2end/liger_examples/huggingface/run_qwen2_vl.sh b/end2end/liger_examples/huggingface/run_qwen2_vl.sh new file mode 100644 index 0000000..963600f --- /dev/null +++ b/end2end/liger_examples/huggingface/run_qwen2_vl.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +torchrun --nnodes=1 --nproc-per-node=4 training_multimodal.py \ + --model_name "Qwen/Qwen2-VL-7B-Instruct" \ + --bf16 \ + --num_train_epochs 1 \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --eval_strategy "no" \ + --save_strategy "no" \ + --learning_rate 6e-6 \ + --weight_decay 0.05 \ + --warmup_ratio 0.1 \ + --lr_scheduler_type "cosine" \ + --logging_steps 1 \ + --include_num_input_tokens_seen \ + --report_to none \ + --fsdp "full_shard auto_wrap" \ + --fsdp_config config/fsdp_config.json \ + --seed 42 \ + --use_liger True \ + --output_dir multimodal_finetuning diff --git a/end2end/liger_examples/huggingface/training.py b/end2end/liger_examples/huggingface/training.py new file mode 100644 index 0000000..5778a94 --- /dev/null +++ b/end2end/liger_examples/huggingface/training.py @@ -0,0 +1,82 @@ +from dataclasses import dataclass + +import datasets +import torch +import transformers + +from callback import EfficiencyCallback +from trl import DataCollatorForCompletionOnlyLM +from trl import SFTTrainer + +from liger_kernel.transformers import AutoLigerKernelForCausalLM + + +@dataclass +class CustomArguments: + model_name: str = "meta-llama/Meta-Llama-3-8B" + dataset: str = "tatsu-lab/alpaca" + max_seq_length: int = 512 + use_liger: bool = False + + +def formatting_prompts_func(example): + return example["text"] + + +def train(): + parser = transformers.HfArgumentParser((transformers.TrainingArguments, CustomArguments)) + training_args, custom_args = parser.parse_args_into_dataclasses() + tokenizer = transformers.AutoTokenizer.from_pretrained( + custom_args.model_name, + padding_side="left", + truncation_side="left", + cache_dir="/scratch/jlee436/liger/model" + ) + tokenizer.pad_token = tokenizer.eos_token + + dataset = datasets.load_dataset(custom_args.dataset, cache_dir="/scratch/jlee436/liger/data")["train"].train_test_split(test_size=0.1) + train_dataset = dataset["train"] + eval_dataset = dataset["test"] + response_prompt = tokenizer.encode("### Response:\n", add_special_tokens=False) + collator = DataCollatorForCompletionOnlyLM( + tokenizer=tokenizer, + response_template=response_prompt, + pad_to_multiple_of=16, + ) + + if custom_args.use_liger: + model = AutoLigerKernelForCausalLM.from_pretrained( + custom_args.model_name, + trust_remote_code=True, + use_cache=False, + torch_dtype=torch.bfloat16, + cache_dir="/scratch/jlee436/liger/model", + # These args will get passed to the appropriate apply_liger_kernel_to_* function + # to override the default settings + # cross_entropy=True, + # fused_linear_cross_entropy=False, + ) + else: + model = transformers.AutoModelForCausalLM.from_pretrained( + custom_args.model_name, + trust_remote_code=True, + use_cache=False, + torch_dtype=torch.bfloat16, + cache_dir="/scratch/jlee436/liger/model", + ) + + trainer = SFTTrainer( + model=model, + args=training_args, + data_collator=collator, + max_seq_length=custom_args.max_seq_length, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + formatting_func=formatting_prompts_func, + callbacks=[EfficiencyCallback()], + ) + trainer.train() + + +if __name__ == "__main__": + train() diff --git a/end2end/liger_examples/huggingface/training_multimodal.py b/end2end/liger_examples/huggingface/training_multimodal.py new file mode 100644 index 0000000..62c0954 --- /dev/null +++ b/end2end/liger_examples/huggingface/training_multimodal.py @@ -0,0 +1,176 @@ +import os + +from dataclasses import dataclass + +import datasets +import torch +import transformers + +from datasets import Image as ImageFeature +from trl import SFTTrainer, SFTConfig +import triton.profiler as proton +from liger_kernel.transformers import monkey_patch + + +@dataclass +class CustomArguments: + model_name: str = "Qwen/Qwen2-VL-2B-Instruct" + dataset: str = "HuggingFaceM4/the_cauldron" + dataset_subset: str = "ai2d" + dataset_split: str = "train" + max_seq_length: int = 512 + dataset_text_field: str = "texts" + use_liger: bool = False + + +def construct_model_and_processor(model_name: str, use_liger: bool) -> torch.nn.Module: + if "Qwen2-VL" in model_name: + from transformers import Qwen2VLForConditionalGeneration + + # These settings are used to reduce the memory footprint of the Qwen2-VL model, + # which supports training/inferences on images in their native resolution. Large + # images -> many visual tokens (a max of 16384) -> large memory consumption. + # If fine-tuning for a real-world application, consider these values carefully. + min_visual_tokens_per_image = 256 + max_visual_tokens_per_image = 256 + + processor = transformers.AutoProcessor.from_pretrained( + model_name, + padding_side="left", + truncation_side="left", + min_pixels=min_visual_tokens_per_image * 28 * 28, # patch size is 14x14 + max_pixels=max_visual_tokens_per_image * 28 * 28, # 4 patches / token + cache_dir="/scratch/jlee436/liger/model", + ) + processor.tokenizer.pad_token = processor.tokenizer.eos_token + image_token_id = processor.tokenizer.convert_tokens_to_ids("<|image_pad|>") + + if use_liger: + print("Applying Liger Kernel to Qwen2-VL model") + monkey_patch.apply_liger_kernel_to_qwen2_vl( + # These args can be used to override the default Liger settings + # cross_entropy=True, + # fused_linear_cross_entropy=False, + ) + + model = Qwen2VLForConditionalGeneration.from_pretrained( + pretrained_model_name_or_path=model_name, + use_cache=False, + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=True, + attn_implementation="sdpa", + cache_dir="/scratch/jlee436/liger/model" + ) + return model, processor, image_token_id + + raise NotImplementedError(f"Model {model_name} not supported") + + +def _validate_and_extract_the_cauldron(examples) -> dict[str, list]: + batch_texts = [] + batch_images = [] + for images, texts in zip(examples["images"], examples["texts"]): + if not images: + raise ValueError("No image found in example from the_cauldron dataset") + if len(images) > 1: + raise ValueError("Only one image per example is supported") + batch_texts.extend(texts) + batch_images.extend([images[0]] * len(texts)) + return {"texts": batch_texts, "images": batch_images} + + +def _format_for_convo(example, tokenizer): + # cauldron data is already in message format {"user": ..., "assistant": ...} + text = example["texts"] + messages = [ + { + "role": "user", + "content": [{"type": "image"}, {"type": "text", "text": text["user"]}], + }, + {"role": "assistant", "content": [{"type": "text", "text": text["assistant"]}]}, + ] + text = tokenizer.apply_chat_template(messages, tokenize=False) + # print({"texts": text}) + return {"text": text} + + +def train(): + parser = transformers.HfArgumentParser((transformers.TrainingArguments, CustomArguments)) + training_args, custom_args = parser.parse_args_into_dataclasses() + training_args.max_steps = 100 + training_args.remove_unused_columns = False # required to not drop the image column + training_args.dataset_kwargs = {"skip_prepare_dataset": True, } + training_args.dataset_text_field = custom_args.dataset_text_field + training_args.max_seq_length = custom_args.max_seq_length + training_args.logging_dir = "/scratch/jlee436/liger/logs" + training_args.output_dir = "/scratch/jlee436/liger/checkpoints" + + model, processor, image_token_id = construct_model_and_processor(custom_args.model_name, custom_args.use_liger) + + dataset = ( + datasets.load_dataset( + custom_args.dataset, + custom_args.dataset_subset, + split=custom_args.dataset_split, + cache_dir="/scratch/jlee436/liger/data" + ) + .map( + _validate_and_extract_the_cauldron, + batched=True, + num_proc=min(os.cpu_count(), 16), + desc="Extracting text and images", + ) + .map( + _format_for_convo, + fn_kwargs={"tokenizer": processor.tokenizer}, + desc="Formatting for convo", + ) + .cast_column("images", ImageFeature()) + .train_test_split(test_size=0.1) + ) + + train_dataset = dataset["train"] + eval_dataset = dataset["test"] + + def collate_fn(examples): + """ + Taken directly from the TRL documentation with minor modifications: + https://huggingface.co/docs/trl/en/sft_trainer#a-custom-collator-for-processing-multi-modal-data + + Modifications: + 1. `apply_chat_template` is used to preprocess the texts before training begins (see above) + 2. `example["messages"]` -> `example["texts"]` to conform with the_cauldron dataset schema + 3. Ignoring image tokens in the loss computation + """ + # Get the texts and images + texts = [example["text"] for example in examples] + images = [example["images"] for example in examples] + + # Tokenize the texts and process the images + batch = processor(text=texts, images=images, return_tensors="pt", padding=True) + + # The labels are the input_ids, and we mask the padding tokens in the loss computation + labels = batch["input_ids"].clone() + labels[labels == processor.tokenizer.pad_token_id] = -100 + + # Ignore the image token index in the loss computation + labels[labels == image_token_id] = -100 + batch["labels"] = labels + + return batch + + trainer = SFTTrainer( + model=model, + args=training_args, + data_collator=collate_fn, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + tokenizer=processor.tokenizer, + # callbacks=[EfficiencyCallback()], + ) + with proton.scope("trainer"): + trainer.train() + + +if __name__ == "__main__": + train() diff --git a/end2end/liger_examples/lightning/README.md b/end2end/liger_examples/lightning/README.md new file mode 100644 index 0000000..916d79d --- /dev/null +++ b/end2end/liger_examples/lightning/README.md @@ -0,0 +1,21 @@ +# Liger-Kernel Example with Lightning Trainer + +## How to Run +```bash +pip install -r requirements.txt + +# For single L40 48GB GPU +python training.py --model Qwen/Qwen2-0.5B-Instruct --num_gpu 1 --max_length 1024 + +# For 8XA100 40GB +python training.py --model meta-llama/Meta-Llama-3-8B --strategy deepspeed +``` + +**Notes** +1. The example uses Llama3 model that requires community license agreement and HuggingFace Hub login. If you want to use Llama3 in this example, please make sure you have done the followings: + * Agree on the community license agreement https://huggingface.co/meta-llama/Meta-Llama-3-8B + * Run `huggingface-cli login` and enter your HuggingFace token +2. The default hyperparameters and configurations for gemma works on single L40 48GB GPU and config for llama work on single node with 8xA100 40GB GPUs. For running on device with less GPU RAM, please consider reducing the per-GPU batch size and/or enable `CPUOffload` in FSDP. + + + \ No newline at end of file diff --git a/end2end/liger_examples/lightning/requirements.txt b/end2end/liger_examples/lightning/requirements.txt new file mode 100644 index 0000000..dbba38e --- /dev/null +++ b/end2end/liger_examples/lightning/requirements.txt @@ -0,0 +1,8 @@ +lightning +transformers +trl +liger-kernel +torch +triton +deepspeed +tf-keras \ No newline at end of file diff --git a/end2end/liger_examples/lightning/training.py b/end2end/liger_examples/lightning/training.py new file mode 100644 index 0000000..ce7cd96 --- /dev/null +++ b/end2end/liger_examples/lightning/training.py @@ -0,0 +1,288 @@ +import argparse +import math +import os + +from dataclasses import _MISSING_TYPE +from dataclasses import dataclass + +import datasets +import lightning.pytorch as pl +import torch +import transformers + +from lightning.pytorch.strategies import DeepSpeedStrategy +from lightning.pytorch.strategies import FSDPStrategy +from torch.distributed.fsdp import BackwardPrefetch +from torch.distributed.fsdp import MixedPrecision +from torch.utils.data import DataLoader +from transformers.models.llama.modeling_llama import LlamaDecoderLayer +from transformers.models.qwen2.modeling_qwen2 import Qwen2DecoderLayer +from trl import DataCollatorForCompletionOnlyLM +from torch.profiler import profile, record_function, ProfilerActivity +import triton.profiler as proton + +from liger_kernel.transformers import AutoLigerKernelForCausalLM +from liger_kernel.utils import infer_device + +_RETAIN_COLUMNS = {"input_ids", "attention_mask", "labels"} +QUESTION = "" +CHOICES = "" + + +@dataclass +class Args: + model: str = "Qwen/Qwen2-0.5B-Instruct" + data: str = "cais/mmlu" + output_dir: str = "mmlu_finetuning" + max_length: int = 2048 + # for llam3 8B model, deepspeed will OOM with 16 on 8XA100 80G and 8 will OOM with 8XA100 40G + batch_size: int = 4 + lr: float = 6e-6 + weight_decay: float = 0.05 + warmup_ratio: float = 0.1 + seed: int = 42 + strategy: str = "auto" + num_gpu: int = None + + +def warmup_cosine_schedule(warmup_steps, total_steps, min_lr=0): + def lr_lambda(current_step): + if current_step < warmup_steps: + # Linear warmup + return float(current_step) / float(max(1, warmup_steps)) + else: + # Cosine annealing + progress = float(current_step - warmup_steps) / float(max(1, total_steps - warmup_steps)) + return max(min_lr, 0.5 * (1 + math.cos(math.pi * progress))) + + return lr_lambda + + +def parse_args() -> Args: + parser = argparse.ArgumentParser() + for k, v in Args.__dataclass_fields__.items(): + parser.add_argument(f"--{k}", type=v.type, default=v.default) + parsed = parser.parse_args() + return Args(**{k: v for k, v in vars(parsed).items() if not isinstance(v, _MISSING_TYPE)}) + + +class LanguageModel(pl.LightningModule): + def __init__(self, args: Args, tokenizer): + super().__init__() + self.args = args + self.tokenizer = tokenizer + self.model = None + self.profiler = "" + + def configure_model(self): + # https://lightning.ai/docs/pytorch/stable/advanced/model_parallel/fsdp.html#speed-up-model-initialization + if self.model is not None: + return + self.model = AutoLigerKernelForCausalLM.from_pretrained( + self.args.model, use_cache=False, ignore_mismatched_sizes=True, cache_dir="/scratch/jlee436/liger/model" + ) + if self.args.strategy == "deepspeed": + self.model.train() + self.model.gradient_checkpointing_enable() + + def forward(self, input_ids, attention_mask, labels=None, **kwargs): + with proton.scope("forward"): + return self.model(input_ids=input_ids, attention_mask=attention_mask, labels=labels, **kwargs) + + def training_step(self, batch): + with proton.scope("training_step"): + outputs = self.model( + input_ids=batch["input_ids"], + attention_mask=batch["attention_mask"], + labels=batch["labels"], + ) + loss = outputs.loss + self.log_dict( + {"train_loss": loss}, + on_step=True, + on_epoch=True, + prog_bar=True, + logger=True, + rank_zero_only=True, + sync_dist=False, + ) + return loss + + def validation_step(self, batch): + with proton.scope("validation_step"): + outputs = self.model( + input_ids=batch["input_ids"], + attention_mask=batch["attention_mask"], + labels=batch["labels"], + ) + loss = outputs.loss + self.log_dict( + {"val_loss": outputs.loss}, + on_step=True, + on_epoch=True, + prog_bar=True, + logger=True, + rank_zero_only=True, + sync_dist=True, + ) + return loss + + def configure_optimizers(self): + optimizer = torch.optim.AdamW( + self.parameters(), + lr=self.args.lr, + weight_decay=self.args.weight_decay, + fused=True, + ) + lr_lambda = warmup_cosine_schedule( + warmup_steps=self.trainer.estimated_stepping_batches * self.args.warmup_ratio, + total_steps=self.trainer.estimated_stepping_batches, + min_lr=0, + ) + lr_scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) + return { + "optimizer": optimizer, + "lr_scheduler": {"scheduler": lr_scheduler, "interval": "step"}, + } + + +class DataModule(pl.LightningDataModule): + def __init__(self, tokenizer, args: Args): + super().__init__() + self.args = args + self.tokenizer = tokenizer + self.response_template_str = " " + response_prompt = tokenizer.encode(f"{self.response_template_str}", add_special_tokens=False) + self.collator = DataCollatorForCompletionOnlyLM( + tokenizer=tokenizer, + response_template=response_prompt, + pad_to_multiple_of=16, + ) + + def formatting_func(self, example): + output_texts = [] + for i in range(len(example["question"])): + choices = "" + for j in range(len(example["choices"][i])): + choices += f"{j + 1}. {example['choices'][i][j]}; " + s = "Below is a question and multiple choice answers, choices separated by a semicolon. Please select the best answer for the question. " + s += f"{QUESTION}{example['question'][i]} " + s += f"{CHOICES}{choices} " + s += f"{self.response_template_str}{example['answer'][i]}" + output_texts.append(s) + return output_texts + + def tokenize(self, example): + outputs = self.tokenizer( + self.formatting_func(example), + truncation=True, + padding=False, + max_length=self.args.max_length, + ) + return { + "input_ids": outputs["input_ids"], + "attention_mask": outputs["attention_mask"], + } + + def setup(self, stage) -> None: + dataset = datasets.load_dataset(self.args.data, "auxiliary_train", cache_dir="/scratch/jlee436/liger/data") + flattened_data = [ + { + "answer": x["train"]["answer"], + "choices": x["train"]["choices"], + "question": x["train"]["question"], + "subject": x["train"]["subject"], + } + for x in dataset["train"] + ] + dataset = datasets.Dataset.from_list(flattened_data) + dataset = dataset.train_test_split(test_size=4096, seed=self.args.seed) + train_dataset, val_dataset = dataset["train"], dataset["test"] + self.train_dataset = train_dataset.map( + self.tokenize, + remove_columns=list(set(train_dataset.column_names) - _RETAIN_COLUMNS), + batched=True, + batch_size=1, + num_proc=4, + ) + self.val_dataset = val_dataset.map( + self.tokenize, + remove_columns=list(set(val_dataset.column_names) - _RETAIN_COLUMNS), + batched=True, + batch_size=1, + num_proc=4, + ) + + def train_dataloader(self): + return DataLoader( + self.train_dataset, + batch_size=self.args.batch_size, + collate_fn=self.collator, + ) + + def val_dataloader(self): + return DataLoader( + self.val_dataset, + batch_size=self.args.batch_size, + collate_fn=self.collator, + ) + + +def train(): + args = parse_args() + pl.seed_everything(args.seed) + os.makedirs(args.output_dir, exist_ok=True) + + if "Meta-Llama-3-8B" in args.model: + layers = {LlamaDecoderLayer} + elif "Qwen2" in args.model: + layers = {Qwen2DecoderLayer} + else: + layers = {} + raise Warning(f"Unimplemented layer wrap policy for {args.model} in this example") + + if args.strategy == "fsdp": + strategy = FSDPStrategy( + auto_wrap_policy=layers, + sharding_strategy="FULL_SHARD", + backward_prefetch=BackwardPrefetch.BACKWARD_PRE, + sync_module_states=True, + activation_checkpointing_policy=layers, + mixed_precision=MixedPrecision(param_dtype=torch.bfloat16, reduce_dtype=torch.bfloat16), + forward_prefetch=True, + ) + precision = None + elif args.strategy == "deepspeed": + strategy = DeepSpeedStrategy(stage=3) + precision = "bf16-mixed" + elif args.strategy == "ddp": + strategy = "ddp" + precision = "bf16-true" + else: + strategy = "auto" + precision = "bf16-true" + + device = infer_device() + with proton.scope("trainer"): + trainer = pl.Trainer( + accelerator=device, + strategy=strategy, + devices=(getattr(torch, device).device_count() if args.num_gpu is None else args.num_gpu), + default_root_dir=args.output_dir, + log_every_n_steps=1, + max_epochs=1, + precision=precision, + ) + + tokenizer = transformers.AutoTokenizer.from_pretrained(args.model, padding_side="left", truncation_side="left") + tokenizer.pad_token = tokenizer.eos_token + data_module = DataModule( + tokenizer=tokenizer, + args=args, + ) + model = LanguageModel(args=args, tokenizer=tokenizer) + trainer.fit(model, datamodule=data_module) + + +if __name__ == "__main__": + train() diff --git a/end2end/liger_examples/medusa/README.md b/end2end/liger_examples/medusa/README.md new file mode 100644 index 0000000..6d0ba99 --- /dev/null +++ b/end2end/liger_examples/medusa/README.md @@ -0,0 +1,72 @@ +# Liger-Kernel Example with Medusa + +Medusa is a simple framework that democratizes the acceleration techniques for LLM generation with multiple decoding heads. [[repo](https://arxiv.org/abs/2401.10774)], [[paper](https://arxiv.org/abs/2401.10774)] + +During training, Medusa requires adding \(k\) decoding heads to the hidden states right before the regular LM head \(h_t\). The \(k\)-th head is used to predict the token in the \((t + k + 1)\)-th position of the next tokens (the original language model head is used to predict the \((t + 1)\)-th position). + +The Liger fused CE kernel is highly effective in this scenario, eliminating the need to materialize logits for each head, which usually consumes a large volume of memory due to the extensive vocabulary size (e.g., for LLaMA-3, the vocabulary size is 128k). The introduction of multiple heads can easily lead to OOM (Out of Memory) issues. However, thanks to the efficient Liger fused CE, which calculates the gradient in place and doesn't materialize the logits, we have observed very effective results. This efficiency opens up more opportunities for multi-token prediction research and development. + + +# Instructions to Run the Training Script + +``` +git clone git@github.com:linkedin/Liger-Kernel.git +cd {PATH_TO_Liger-Kernel}/Liger-Kernel/ +pip install -e . +cd {PATH_TO_Liger-Kernel}/Liger-Kernel/examples/medusa +pip install -r requirements.txt +sh scripts/llama3_8b_medusa.sh +``` + +**Notes** +1. This example uses an optional `use_liger` flag. If true, it does a monkey patch to apply liger kernel with medusa heads. +2. The example uses Llama3 model that requires community license agreement and HuggingFace Hub login. If you want to use Llama3 in this example, please make sure you have done the followings: + * Agree on the community license agreement https://huggingface.co/meta-llama/Meta-Llama-3-8B + * Run `huggingface-cli login` and enter your HuggingFace token +3. The default hyperparameters and configurations work on single node with 8xA100 GPUs. For running on device with less GPU RAM, please consider reducing the per-GPU batch size and/or enable `CPUOffload` in FSDP. +4. We are using a smaller sample of shared GPT data primarily to benchmark performance. The example requires hyperparameter tuning and dataset selection to work effectively, also ensuring the dataset has the same distribution as the LLaMA pretraining data. Welcome contribution to enhance the example code. + + +# Memory Profiling Result + +> **Note:** +> 1. Benchmark conditions: LLaMA 3-8B, Batch Size = 6, Data Type = bf16, Optimizer = AdamW, Gradient Checkpointing = True, Distributed Strategy = FSDP1 on 8 A100s. + +## Stage1 + +Stage1 refers to Medusa-1 where the backbone model is frozen and only weights of LLM heads are updated. + +``` +# Modify this flag in llama3_8b_medusa.sh to True enables stage1 +--medusa_only_heads True +``` + +### num_head = 3 + +![Memory](./docs/images/Memory_Stage1_num_head_3.png) +![Throughput](./docs/images/Throughput_Stage1_num_head_3.png) + +### num_head = 5 + +![Memory](./docs/images/Memory_Stage1_num_head_5.png) +![Throughput](./docs/images/Throughput_Stage1_num_head_5.png) + +## Stage2 + +``` +# Modify this flag to False in llama3_8b_medusa.sh enables stage2 +--medusa_only_heads False +``` + +Stage2 refers to Medusa-2 where all the model weights are updated incuding backbone model and llm heads. + +### num_head = 3 + +![Memory](./docs/images/Memory_Stage2_num_head_3.png) +![Throughput](./docs/images/Throughput_Stage2_num_head_3.png) + +### num_head = 5 + +![Memory](./docs/images/Memory_Stage2_num_head_5.png) +![Throughput](./docs/images/Throughput_Stage2_num_head_5.png) + diff --git a/end2end/liger_examples/medusa/callback.py b/end2end/liger_examples/medusa/callback.py new file mode 100644 index 0000000..673243b --- /dev/null +++ b/end2end/liger_examples/medusa/callback.py @@ -0,0 +1,386 @@ +import os +import time + +from dataclasses import dataclass + +import torch +import transformers + +from accelerate.utils.constants import FSDP_SHARDING_STRATEGY +from transformers import TrainerControl +from transformers import TrainerState +from transformers import TrainingArguments + +from liger_kernel.utils import infer_device + +# https://simple.wikipedia.org/wiki/Byte +# For memory, we use binary system +M_BIN_UNIT = 2**20 +# For metrics (tflops), we use decimal system +T_DEC_UNIT = 10**12 + + +def round_to_n_decimal(x, n): + return round(x, n) + + +@dataclass +class Precision: + """ + Precision is a dataclass to store the number of decimal points for each metric. + """ + + n_decimal_time: int + n_decimal_memory: int + n_decimal_TPS: int + n_decimal_MFU: int + + +@dataclass +class State: + """ + State is a dataclass to store the internal state of the efficiency callback. + """ + + n_warmup_steps: int = 0 + total_peak_memory_allocated: float = float("-inf") + total_peak_memory_reserved: float = float("-inf") + + step_start_time: float = 0.0 + elapsed_time: float = 0.0 + + elapsed_step: int = 0 + + step_start_tokens_seen: int = 0 + elapsed_tokens_seen: int = 0 + + step_start_flos: float = 0.0 + elapsed_flos: float = 0.0 + + global_start_step: int = 0 + + +@dataclass +class Time: + """ + Time is a dataclass to store the time-related metrics. + """ + + step: int = 0 + step_time_sec: float = 0.0 + avg_step_time_sec: float = 0.0 + time_to_completion_sec: float = 0.0 + estimated_total_time_sec: float = 0.0 + + +@dataclass +class Memory: + """ + Memory is a dataclass to store the memory-related metrics. + """ + + step_peak_memory_allocated_MB: float = 0.0 + total_peak_memory_allocated_MB: float = 0.0 + + +@dataclass +class TPS: + """ + TPS is a dataclass to store the tokens per second metrics. + """ + + step_tokens_per_second: float = 0.0 + avg_tokens_per_second: float = 0.0 + + +@dataclass +class MFU: + """ + MFU is a dataclass to store the MFU metrics. + """ + + step_MFU: float = 0.0 + avg_MFU: float = 0.0 + + +class EfficiencyCallback(transformers.TrainerCallback): + """ + EfficiencyCallback is a callback to track the efficiency of the training process. + The tracked stats include: step time, memory, throughput, and MFU. + + It requires including `--include_num_input_tokens_seen` and `logging_steps=1` in the training arguments. + + Args: + n_warmup_steps: number of warmup steps + The stats in the first n_warmup_steps will not be added into the aggregated stats + This is because the first few steps might take longer due to jit compliation and other initialization overheads + n_decimal_time: number of decimal points for time + n_decimal_memory: number of decimal points for memory + n_decimal_TPS: number of decimal points for TPS + n_decimal_MFU: number of decimal points for MFU in percentage + """ + + def __init__( + self, + n_warmup_steps=2, + n_decimal_time=2, + n_decimal_memory=2, + n_decimal_TPS=2, + n_decimal_MFU=4, + ): + self.state = State( + n_warmup_steps, + ) + + self.precision = Precision( + n_decimal_time, + n_decimal_memory, + n_decimal_TPS, + n_decimal_MFU, + ) + + self.time = Time() + self.memory = Memory() + self.tps = TPS() + self.mfu = MFU() + self.device = infer_device() + + def on_init_end( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ): + """ + Event called at the end of the initialization of the [`Trainer`]. + """ + if not args.include_num_input_tokens_seen: + raise Exception( + 'Please pass training argument "--include_num_input_tokens_seen" to track tokens per second' + ) + if args.logging_steps != 1: + raise Exception("Please set logging_steps=1 to track the efficiency metrics accurately") + + def on_train_begin( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ): + # if loaded from checkpoints, global_start_step is not 1 but state.global_step + self.state.global_start_step = state.global_step + + def on_log( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + logs: dict[str, float], + **kwargs, + ): + if state.global_step < (self.state.global_start_step + self.state.n_warmup_steps): + return + else: + # spread self.time, self.memory, self.tps, self.mfu to logs + # logs.update(self.time.__dict__) + logs.update(self.memory.__dict__) + logs.update(self.tps.__dict__) + # logs.update(self.mfu.__dict__) + + def on_step_begin( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ): + """ + Event called at the beginning of a training step. If using gradient accumulation, one training step might take + several inputs. + """ + # memory + getattr(torch, self.device).reset_peak_memory_stats() + + # time + self.state.step_start_time = time.perf_counter() + + def on_step_end( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ): + if state.global_step < (self.state.global_start_step + self.state.n_warmup_steps): + # The end the current step_start_tokens_seen and step_start_flos are the start of next iteration + + # tokens + self.state.step_start_tokens_seen = state.num_input_tokens_seen + # flos + self.state.step_start_flos = state.total_flos + return + + # time + current_time = time.perf_counter() + step_time = current_time - self.state.step_start_time + self.state.elapsed_time += step_time + + # step + global_step = state.global_step + self.state.elapsed_step += 1 + avg_step_time = self.state.elapsed_time / self.state.elapsed_step + + self.time.step = global_step + self.time.step_time_sec = round_to_n_decimal(step_time, self.precision.n_decimal_time) + self.time.avg_step_time_sec = round_to_n_decimal(avg_step_time, self.precision.n_decimal_time) + self.time.time_to_completion_sec = round_to_n_decimal( + avg_step_time * (state.max_steps - global_step), + self.precision.n_decimal_time, + ) + self.time.estimated_total_time_sec = round_to_n_decimal( + avg_step_time * state.max_steps, self.precision.n_decimal_time + ) + + # memory + step_peak_memory_allocated = getattr(torch, self.device).memory.max_memory_allocated() + step_peak_memory_reserved = getattr(torch, self.device).memory.max_memory_reserved() + + self.memory.step_peak_memory_allocated_MB = round_to_n_decimal( + step_peak_memory_allocated / M_BIN_UNIT, self.precision.n_decimal_memory + ) + self.state.total_peak_memory_allocated = max(self.state.total_peak_memory_allocated, step_peak_memory_allocated) + self.memory.total_peak_memory_allocated_MB = round_to_n_decimal( + self.state.total_peak_memory_allocated / M_BIN_UNIT, + self.precision.n_decimal_memory, + ) + + self.memory.step_peak_memory_reserved_MB = round_to_n_decimal( + step_peak_memory_reserved / M_BIN_UNIT, self.precision.n_decimal_memory + ) + + self.state.total_peak_memory_reserved = max(self.state.total_peak_memory_reserved, step_peak_memory_reserved) + + self.memory.total_peak_memory_reserved_MB = round_to_n_decimal( + self.state.total_peak_memory_reserved / M_BIN_UNIT, + self.precision.n_decimal_memory, + ) + + # tokens + step_tokens_seen = state.num_input_tokens_seen - self.state.step_start_tokens_seen + + self.state.elapsed_tokens_seen += step_tokens_seen + + self.tps.step_tokens_per_second = round_to_n_decimal( + step_tokens_seen / step_time, + self.precision.n_decimal_TPS, + ) + + self.tps.avg_tokens_per_second = round_to_n_decimal( + self.state.elapsed_tokens_seen / self.state.elapsed_time, + self.precision.n_decimal_TPS, + ) + + # flos + step_flos = state.total_flos - self.state.step_start_flos + self.state.elapsed_flos += step_flos + + # MFU + # 1. Definition + # + # MFU is defined as (achieved TPS) / (theoretical maximum TPS) = (achieved floating point operations per sec) / (theoretical maximum floating point operations per sec) + # Crucially, the "theoretical maximum" throughput only accounts for the required operations to compute the forward+backward passes, and not rematerialization. MFU therefore allows fair comparisons + # between training runs on different systems, as the numerator is simply the observed tokens-per-second, and the denominator is only dependent on the model architecture and published maximum FLOPs for a given system. + # Ref: https://arxiv.org/pdf/2204.02311 + # The benefit of MFU is that it + # + # 2. Implementation in huggingface + # + # current_flos = 6 * estimate_tokens(input_dict) * num_parameters() + # total_flos = sum(current_flos) # across all GPUs + # Ref: https://github.com/huggingface/transformers/blob/616bb11d487aabc231bb230b245c42214ea4b254/src/transformers/modeling_utils.py#L1196 + # + # 3. Derive MFU on rank 0 + # + # rank_0_flos = tatal_flos / n_gpus = measured_flos / effecitve_n_gpus + # rank_0_MFU = rank_0_flos / step_time + # + # For FSDP, num_parameters() is (1 / n_gpus) of the total parameters. So, the effective_n_gpus = 1 + # For HSDP, num_parameters() is (1 / local_world_size) of the total parameters. So, the effective_n_gpus = n_nodes + # For no sharding and zero-2, num_parameters() is the total parameters. So, the effective_n_gpus = n_gpus + + num_gpus = EfficiencyCallback._get_effective_num_gpus() + step_achieved_tflops = step_flos / step_time / num_gpus / T_DEC_UNIT + + avg_achieved_tflops = self.state.elapsed_flos / self.state.elapsed_time / num_gpus / T_DEC_UNIT + + precision_bits = 16 if args.bf16 or args.fp16 else 32 + gpu_peak_tflops = EfficiencyCallback._get_gpu_peak_tflops(precision_bits) + + self.mfu.step_MFU = round_to_n_decimal(step_achieved_tflops / gpu_peak_tflops, self.precision.n_decimal_MFU) + + self.mfu.avg_MFU = round_to_n_decimal(avg_achieved_tflops / gpu_peak_tflops, self.precision.n_decimal_MFU) + + # The end the current step_start_tokens_seen and step_start_flos are the start of next iteration + + # tokens + self.state.step_start_tokens_seen = state.num_input_tokens_seen + # flos + self.state.step_start_flos = state.total_flos + + @staticmethod + def _get_effective_num_gpus(): + # Calculate the number of effective GPUs for the total FLOPs in order to calculate the single GPU FLOP + world_size = int(os.environ.get("WORLD_SIZE", "1")) + + if transformers.utils.strtobool(os.environ.get("ACCELERATE_USE_FSDP", "false")): + sharding_strategy = os.environ.get("FSDP_SHARDING_STRATEGY", FSDP_SHARDING_STRATEGY[0]).upper() + + # Either specified as string or enum number + if sharding_strategy in { + "FULL_SHARD", + str(FSDP_SHARDING_STRATEGY.index("FULL_SHARD") + 1), + }: + return 1 + + elif sharding_strategy in { + "HYBRID_SHARD", + str(FSDP_SHARDING_STRATEGY.index("HYBRID_SHARD") + 1), + }: + return world_size // int(os.environ.get("LOCAL_WORLD_SIZE", 1)) + else: + return world_size + + assert world_size != 0, ( + "WORLD_SIZE should be set to a positive integer. For single GPU training, please explicitly set WORLD_SIZE=1." + ) + + # TODO: add deepspeed support + return world_size + + @staticmethod + def _get_gpu_peak_tflops(precision_bits: int = 16): + if precision_bits not in {16, 32}: + raise Exception(f"Precision bits {precision_bits} is not supported") + + device_name = getattr(torch, infer_device()).get_device_name() + + if "A100" in device_name: + # data from https://www.nvidia.com/en-us/data-center/a100/ + return 312 if precision_bits == 16 else 156 + elif "H100" in device_name: + # data from https://www.nvidia.com/en-us/data-center/h100/ + # NOTE: Specifications are one-half lower without sparsity. + if "NVL" in device_name: + return 1979 if precision_bits == 16 else 989 + elif "PCIe" in device_name: + return 756 if precision_bits == 16 else 378 + else: # for SXM and other variants + return 989 if precision_bits == 16 else 494 + elif "V100" in device_name: + if "NVL" in device_name: + return 125 + else: + return 112 + return None diff --git a/end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_3.png b/end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_3.png new file mode 100644 index 0000000000000000000000000000000000000000..34f044faf2fe47d1cf972bb09e9472f0dae877d7 GIT binary patch literal 13454 zcmeHud0dj|*EiL)Pg&EHHKkV0XftM(m6jXrwuofq9%7j}rbwwkia=@0RFh_lrleFR zmK&~+3y^K5q?U+?C{!+lfTXerh`cv8GoP9H^(>#y`@GNde%^Qfxw(Lw>ps`H&iS5m zeb3?2eoyx~v%Z@J0)gi2*}dZc2sGIb1ez2xV;b3k-ZD|U@dYJ#CHTo=LLNZU3xKj8kW zlmp-X9DY9k(2te-bRoDstR|53omM<>MpLUsH>Z( z7U%@JIJe_LH)=$8u!DmlDGw*(l@k|y0|<3E>+S;JNT3S!o@W!ye9PsbBR-voorU zxHU>{F;-{spanM=BfhXSYPX5o=-uYxj#wg+#Ty>!ZP%G?b)0 zwxpxQ7u$Cq|4!>b}&$#MGE(+#^V6Gc9B*R;H zjHqh;7@w)tI!7FmM)7_G~1lrS`NxK#hgWz+8$}D+;brxYUHl4hy<*wn! z=1wU22%NOFos?c#hA;>S27OOIKlK}8HtCQKCXdAM5j5|Rp-<0DB&*ZcA&60G3^6Kn zi>$M z0#ZHLO%I3ZRvgVxC8By?rJl$(`%qZ!+m4=lzWZLJ;Jj(j?Wo$Jmm3@$9HKVIsJk(~ zl>4%xU4iXEMSRWd_@F~Gt`D&A`# z2RS%QL-VB$ok|*QBXI9{@es9h#nBpsh4#(G);Lj?uj@QVx7dk`5(RhJ-$c>pHpl z8i!`|(#SEXgI*>amz_f?YPyGoWwdf>*K(}xbr7O+Lrpf?;*ihcNJdfl6&Qg}r3$Rf zLpmM0gARAZE+tu67)I+0Bo420{HlUE!fBB=knXN1y>vbA-u8?6ap>NbG+NF@YB#Z+ zFUPfZ$UA$y@Q2$_5bvOL`uRwAq0DU*%T!}I`fB{4av5?!k)S!PBH{a(p&?O{ij_y~ zUdd{=(vfrQ+mj-ZDfP4?yc!G=WhT{HQp4^1RE$!=X6a1#me~zez1R1>bX~z~e!%J- zYZ8t@kehS3&(O_djos)C*!`0F7pCyT`ed_4N2AaB!^Ye4++*e8E{lXS?PIVxXUYY6WFBVp#A}a4J ztg+G{p(u~5p0rb4G$h>ntH|m!>g#L}f{_K?uv+8PaT|(m_GOqUkLd^3zoa7AdP}DF zLAZRmdQXfHr&i+IQD!N%#pDsr_-h)eE z3lXfl((^AwV1s4=gRJUXlyx;2Krertkb%Qe*0!}+@M_~?Ha0aEluSPUB23hrML!xs zd09R-t6gxp1l4I{ja?YZmU?@`^zj`F9^ew+uHnZ8Lnr~69oCzMt4D0WI-^4p;dwNe zors2!s*4`-luS`JHeOP0IhzMIjDUUF(uWt!D$6Wq#933ot2|I11JiiZRF~Efk5htW z4Q=BB!dN{)QS8pv65G-%ttVsooS@3@j1w0`!5o$XDi#`^fybVmSdGxDe&UmPHpg^3 zG9k1f7IY;w(%}`FMPN10%(v(?^gA;BAvEi`C~yNk5J zvV2`x_zn844ZPbAw^FwTwWCL?$Eb%UblK#-NH+6z2NTV6%PwRcPdY3b)A=OZV>jN; z<1+EfnG1agOR?8v?F!nj3m4e9QXvB`SlNrEGPup`8>O^qh;E2zdB*QrJL-;?oj-8H zWe|ac$CsX}$g%P{gys!T=)4eE8 z$~B%BlIX*P)AW$6Y&$5*uw+M29`57{mZXB_RNk{Bf=#8<&4smkHdueqJ6Yz2w7Xe& zOg!J~xjkYkYjB>b_o&EDJU2CEa>Y*gk;T}`V^Fl}L_-i~L-)`~#hB047s)HXUP-AL z-x4Y{zqYqJaSajdz9lniJ064#Al%TF zRXyw|hzzkdbcSNTVTYD)Esn`PZqkQi*v zd=kzor78-GO?}Av!=P@Z1bt^i);cPp`HFth|J8$`Mq# ze*b#xME-cbot-mJnbl~tk+U#&BAHfK9L6{LTa)N3{g8#>6pA-q$x1)M;2Qn0@cfPuaEH=j4=SYawSgY2uZ6rp10vvMuNYbUEOtzt$VAgal&0I zH9v;Mx-i6AzP-0*LAjEKoe%vb@m5RdN73%9bovoiXW%jP+%t3&IQR%{9CR&@cd1Sl z6winhIVdejA#42y+|7q8s4UIyU(7TQS(FPBi*+qMN39oQGA0(hbxlG1(&4)$2{md*3B^ef-#bEKbkm~gI#YLB&ZU29qaskZBT3v{#SXJaNS9DV5+PpcHOe^zE`C2&JL z5ZD%$Tn}@fG3*{I;S#Q`Yq-dfH9U+00Kj@Ne}hdvwXl1d^WNJQUk}h|SJn{gt>E?A z`fDiv^{-G-TK!hCNm;i7L+s$A(9sBdV)=9(zSnf7$_KVU&reDY=f^rfJz_Ed5vFDL zLH7(y-&Wfj!-?#YCfH-ohN#uGw$hy!n<%CwRGkjc)3_q7n<-1M!!9e8T<3aO6l@*5 zv6qW->1@wskC^&*`u@}*E3?ef6!mJqq_YxS!+17svS4qC>>_$;jF?6tZ8^MRR$6y> zCnMz5V6W1!tkdm!5v%P6cwJ~_u{$1h+@w~Ue|L=`i)x5j%d%M540pZF%-mX{FHv zqVL6J$FE7c6uKl2i(@rnxkCeV?jpWbeAGVT*q$vmi$B|pQj91q<(lE^ z%vF`Ps>Fx*4fkc^Zh)t#T>UmHq4(whAvLbb;YJfBBG4y~pB_D+d~Nq(fF<@CxZSoZ0G1six!dA~;Qp4`7mfd=L3=>=qQIG^ymo3XAM?qH=-BbGY6Wdc#6i7jbRo7UL1 zGmqYHF1}&5@FQ)t=Pff^Dk73wubSy>TF=nE+FOu4CPWh~j*%KzeCrnJstZ?h^z<=0 z936BR##Ha&P+iL!`ni?11#H>*B(tH?QziT(eLs=*`NBPcz#0$k#M8p#nIWzpRjSho zBS%vf`plCtxXQ;#r-73=D3&D(;=N62V5A=Ii`gngB(EAaocbAAcXK7-$qK^Wn2y5l zJyeq>gHG))ITL=&qPT>{9gbV060*K-OrFOx=)x%ZIVCyM(9yb*l%S?_35gZByYJuC zC2_ngu#%0W=?4v-9cd|=hLLSHCaxCn^ zJ+VQ^Dy5OFbUVsSbr*&j7-c`KG%GI#mCu|y`}QvM_&FT(2JY!2neL7@loZ8|aX+32 zo36JzaeRR@g~fOwXs&4#ZR}$hUbCxvg$8zr z%6<`vLw`)N12)ZCO?O{V}N-@84W6O57)~-fcrnqN<+ih(EVWU86&4F3(PCZek z$dNTXLc|}+iN;lKfu!MI_aeAVMBjj`JPzuFj?@Z-Z4yxM%2|uvY@OY9Ozh6TaS&I% z<{)#dBP1mkK6)O$+YTq*$2R8u43W4c{c!Xir;*p|*(qbMQXVWt)|NU_BFs<-c2gEc zkRkYV!STV1E2epDDA*N-;&{{J)ItIwz9}N;M$p&?(>!8S^&>}pgEh%Re!AYd_4*I{ zTKNJMwE`(*2x@!oO~@;us%u=UqbRoOOVEUsbX6rq1}SJ^xv8sKa%l`@QFvrzp(Em& z65VZL6=-rM>f6r2jTV#}4L?YmwPQKexJ+74$ zSaS7fafU_NfI} zq|%vtI0h2H#N&{Hp*NF(EqROL>$=ThG;W&isVU=H_UR9o1QlAB7~s%AHbzItbdzK- zGV)(#HNi%o*oVug*7hy5@p%C7D-~J|UpS5yjwnRkH5kWcNq{?ykMc#w=DG8CSe+TGm|wupYwqWH<3o7Lw6=$x~Yyko*L zxCk!4(;*F5?HXFv!O#(beh}jeCK%Z-q66YPN5VDbfNXKEcG5?{Yn*;Qo4H#5Gd4OO zN-@-iwPc@4hmQ}ZtCKIgjJ`|4wM6Lr?Jz|-#fa)SHc@cXP5U;5Ou@+_ID8LfD17*O z`k|>FytmofsvSWUMzf9WR^jW*Bq~oT-X`LufT8PykM}76u1nExQYI*7B^6G8vg1#d z04&#e_|&sGj)z3tjyM-(9J=kp{VQwonzJey@n zVg{>3$$=_>UJFA#Rm*5Xw?TxpE>PVS19eImJ*gc5PN4(e$C(7&wb}=+j>0lScqpt6 z#?-Xa7`Fst{f_Fr+~pEkwacq`kjIX{XraY=842APF1~D2yx7r>VBgdnS{8G%@U>z+ zMVQE`Om>2~XJx$`&-bP0JIy*Od1x0wTIr&bmKQ+VdGm{HDjM8l`YlHeGt>U**y%^ zEJ?3-T4^-o3rDEG{%~g8tK`gFcexfU{^%I=20o7AiGz_ATLIp-Hs6fj)E28@vf4}EPKeS?iL%}Q0 zC5F0ZqU8dZz80rbRYda2{&GuNU{uEbl0|C{-+uY>Wj@Ko%<1N$)qZQ+;sE-4eH}fx zFUNP+?Dnpz7%bSr8NIqGV}01e5#YPyiP*YEcgnAe3U^*vlnc1oAu#`~m9uuk>$nr# zVp#Y{f77{n+h(j-aMZQ#3bwB7KyE{t9XuO7*yXRU4XIDAh!wZ~CdlB_0?(A-BL8x$ zA~@4c22jpLfwVP6LG<{~PkLIhs1P_U=Ni#AZlvFvG+r(*`L5pu1Ui$kwk>fg2(%vn zb>Q9l|Bh?hfX2?WpJv`wN9i~D}6FNQuc+Q{o~0#2@c zaOm6BfU;)VI?g}vlTp0#b7{mGKQ5d)8+68h8w+yo=4D_3PgdXSv)cCCEN`M0tjN6| zJPEXR-_KQ-t{w0y0XF@Iimn~If1ft$zkOPpl8_X^?UzxwpzmtL*yC;ntgBQDlyF)< z%R1FTbyWmi?X*6_Kz9S3dHkEOXCbRL0cp`M`#!I8;cZ~)lR(dNKd*I`2k`Olb41(u z0esV*uBuN`M7)1`-WZs*nm6~h zhu0-e{oCC6_r5K(CMDSGpO}no$#g^pO%z9nZkWL?K}&tI*}jGXK)Op)>-vNwc9z8iV#I=sKe|B0{5~_8N zwQ`LeoZ%wtsw&sU0gnn7x90v$x}VgEq1IW#cuA4?)B9)Q{DR`WV1PjCCen^KWme$z z+JUh`V=;(R;A?^sJY)kK{QHJ>&29%+^3))0YdZM`)NQjdUx5IHrC_9h3?a6DcyXm5 zro8YM9>#A@%-?^xXv`??@B8*`;gdga#w?%UKWTj7RGYQHD*tnKb3b@0`A>TEDw+Ie ziC6iZ{j<6J^VmPk`u}SsH`0T<(R_s1h#l5hu?Hry{PvP_`$bos{^aPuu4KgzJQAN<;JG+Gl1+w$4}EwME~t&;ve9s6hF|K~ttH@Y6@&fZLNS$o7i^HPVhL^1^&Ddb&2IM;|5yBj055qYjj?_3FS z*&^ytp0%LLr7x&i-LVOlKu$1$f4j-f`_DZ|Nk*J|axppL^9V=>CcO6`yiK zmah18ujjkZBOpDQfL;6Uvn#*C$3|LyI20YOivIV04~g7bFXpx2(bdv2)_1eh%)&eASd4N+CF8TpLRl9+r^DC2DX<44~Q2+4_+C*MB>`Ye3(2>}075nPw}TS)jKTx=|zx;gwOzb}Q5fA9V)i){D0 zCog-1r{+M=ityVbZN;#f*8V1RtU zO0&K3!!9G0@o+DL&frF|#xC7A-iI_NbC2o#`8{Z^N*3>O2Ar**MFaAa*q2MfJWii+fEiL% zSArDxV~x#`&T?aeW{{@Q*C_&-D<44D$v!Q#ca;wNXPh=EpG4>`f0!;Y+BjDCm}#EHB|fEL;O{9{Ls$I8NB zDSQ~8CK&>aTSb|D+xJJpgK5JXED`)vnYT|CJ)b)L99env_(ybRq>hSs2DH_d2BeqD zN#(xJZZARbA4c%qB>pV}2rzcoO;*8+FUKDdGNhsGS3_|FH zb2-Dj6gYk7iB2z_KBYo7i8nViOygY{9$PfI-Jg|jo+EoGUxq__SYXf zcft7W$$NPY6_#cb{i(G3_BviO-yZ$#^R&9xAH%?GSyVL@*{(PZA}pZIz1&uj8zrCNf~TvRyCCp#nd_!GW8Ec7lO4PRgaKB)r>n^#7tv-qi+aZv zDZQlEF^%T17U-R6vBE}jWD%X_ChsU%$=$^e=M!AQ1B8Vdr8+@9&l!w52K2{r>k+u-7JQFTjT8Ff?tgL?YCHiy^9AcN zqA=`ATn(_Ql(#^M;=i)K{=YNZ+HVGH*dCh<8G{a5Q2#E)fH;&pTkQ`a4C7}tBBEx$hXZlo;5 z3N-IkPKsM5UdF}SW}SMuB)s{~W^o1Eb00h>MrxQL+ zFBlF1{!h>z@Su}SELQU6Jbs#z1!{bm13HsBV(cQAZCAUNKKjCL5TX;v^@9A4-q$2B z)3F6lJ#cKUg($p4dH5)YK8ro;ZLhT!QXfAlZ<8l&;J?l6J=q7l$CRM6oZzMhs2K% zISq1>=5<)BP0-L~L2YFO*M}{A9;@p;5;*Au(3Q|mVdkYzx_Qxk63DVHIf&k4CV%L* z`7}^H$aSoJKPMch-qAH!m2?j>p=T-k>G|0c)fa#)WD~JJ$Dpu3r_rBjp&3V literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_5.png b/end2end/liger_examples/medusa/docs/images/Memory_Stage1_num_head_5.png new file mode 100644 index 0000000000000000000000000000000000000000..61d66f6b0465c4b4cf83013e69676eb0ed4dd128 GIT binary patch literal 13939 zcmeHudstFwyEkP{O-xW#O=FeR=7X$*$-@k9q5fEsmCkQknY|b~pH~qZGN)X7#ZvUR0$1Xy}Y)s_} zM2u$aUhegU7jkY9jAR?B_`IR3taL$x*Eeo?~Wba<1b#~ z9IT4%&R=;f$fRiB;TApC!}eVd)^1#~>_ErrUl*>io;7p-+dNKiRVaa!U)GWgV26lHGKE&!RrdD9Y7nZ29qL2g^XXjk-$kW}% zH3Gg|{mw{za;jgSa<{lk9i`@Wooj?5h_YBnrgq=k>{uqj3;R(lLSxl zvNOZkJGor@nD@)MLr5-iy^wEl8s9cfvR4eG9F;d*Sjf7ur0{}h#u!G;$BZ-kkeqKd zVy+t20eg&U@1i_O4`dlTj9rf{hBy+5+q30seH!rTC0C6mITLm1H2aCZjmqSvyJ`F& zjz&F^H;SI9Nh#KRc%i4ix2Y?r-4G=zv$nLf3?N@{ke4MrGS0sEFiZtZh#bPLBVYX9 z?mMB~S)}*R)(jybm|47WVhotjj(P4q42UDv5IK-M9*B_#Hq=thDbXF%J0BlU5|n!= z35AqtD#cV?gpzc8oZ)?D9*)NsW=C?6d%d}R;X#?|JVi2SK7x_jVp1Fw&5)%`gIvl~0cBIyySRCq_f2pynDc9Upghe4 zMiZIm+(SAY<-MB3&ql?=EP*L6TAlR5xrBw8Zo(|al#D38DQokrZxM-iW_ifEekI~% zLs!Y`PL^zToUGTwU!JYERMQ)fRXh$K=JnVN$pnE41<9Qt<|;>O;2qK6Bc~YBlyfE2 z7<`Sa%R8a$;?RSu;c3*nT`ojrjSJE8e#9upr?mpHd@ZV=L!C(WJZfWTLwDW#YB==Z zVqeV^Ga)AqGnI!~hV}_OFmV2^0HNdqtJy!Q0v57}_I%;pvA8WZHtwxT6<4F$&k(0p zVRx6Ef8ThDqu}Fk#T50FwS$9$AZidm;wI=`miNsFA5>K+-+Typ*iCqnf4bYGPY*Ns zLeLvrg0(K8X|=zY+U&%xlcbg`PKS}St8nvXEZe5$CP6JHG}=-WZ7r;G{5#z+<&cwO zCT$!4UVrS<;l500=}TOfFVAwX>a$ajh+H&kY!sLy7m^82 zhfTqVZ1p&fsL3j39+^f5KFxYiUkuP3pN74tz) z{Z=ntiBLQ~!{)vbC?v@uM{)~>+Sr#CbcF-G#X??=;tmHEDtkDFNpM365~&%7XmTrl znsuhuPuy2saW^n>q&54O@!DJW5;l|0b}YIrc8Tg?Yd<{i zj}#rMQ_wZj^nOX8qtdH-@C~4E@r-w2IAz}*1hG5i7sZU1hHFJIeR8(_j&5%ouM+=< zQd2KS*0ZkHp*2vt>(eftO*6BNx2wrYGauZPcdlAh7?vIS{sFT}Drj+9c%g-tixa2i zvFy4j5x+iHhpJW4j+<%}gINhk_5lq6#)=nul@;99$iti7423UmQYk0&_ddf12T?`e zwhyutdoJH*);HF&w)rX|?xe^_^nb7jZWBq-q#ej=RiAMid4j`Ic%>uOO3L$T9?Gkj z$*UWgNA2u=vq#j7q-mw1^2UW0A&2fcQA17|A9KMFH?yY|I1Tc=y@{vqR7ZOJ!;l7f zi%E$o^utQRk0zd7>PS)d>y6*>q)vp8o+Hi`(oKbzyvE8>lo!g%Jl=bub%Wu1yDi13 zDdhO_AbEa%VXdKjRuImwk{V&?*;$ab2{kbyHU(28fPF3p>~265OLhT`YC?14GmA9r zNG{te+raCUIvZe zGM5AkfKFRQ58_^%&VNzviY&WTil#VZgFjh|yXpt^Z2mW(rQ3+Yw-J zz1Flwua`?CMoM2V1o!6rVFidG8S<_QGe*2ycACtpOBDi7Y9dCHLj~&{eFyb+-0^1- zn0$-Y#Mq^Uq;3P*YbPV|2}erIL)^8NlIP-7OiJ`)PU7M_JW8N6%%W^IA3bu%w(pnS zV<>t2gVyY`A*42QDyCxOffV^=n}yv!rc%c1`y#4zJ#vWrmeeuQXoSQTQpHaQ)C+M51-##UO?Mv$vdA(yPKBh(U3UZJMYmZWS@{zH-75< zStpmm$GuN^h>E7vwk%r+DU=N>JVK-otl7x>0VX-zB#|hqA)9=4;yfh~N_WwCM3SoCSMNSh<%L z#5!W0naBy91CXqPB5gj$5tI$HDxZcI1cdMuSR@qf;e_u)}2@J&msF zDyn=fp4m3##ot~O?aAisdc~k}amTW=Wfg46sC&$u)s!FTM8*K|p11}}D)ZGUiTTQl1xy^6vYDUA?J3PDXwXKSHLhOK}LFru-x=cp@pNi)M!k!B{O) zbiAFi@WlGAicsy&Yva#SHRGoDpYPEMzcdzVD-6U+t?g&y&sDoIls%yc47Hn39^kc# z>U2cPLd$KOY?1cE9K1=%M0jIm5UGsV7?PdM3Xm+ii0msE-~}fhL%%-i5a&)5q2(E& zth#6wdL-olV8Djc9x-I)Pc~U8$~k$mC^mlL;YcJR%F&-qm&-3kGkDAZQodp2Lddna zRK3gc3%QLPPN}fh`*g&DOJqG?hm*wNmuJ8h-JFB;ba5XZIuQ09=~48+Do8=iy`*2e zlatD)&pn;~Ub=mAMBMol!Hw&~fy)QV^NID)u}(8eJk*wQq18x1ENc-KGq(U%ymNHI zii{KEDFqU_)&a|cLdpM0vcQ{HEOJ7sJK>o3Jnkzk>dd1%8dLb!$I5Nt7810ca@dw+GQ9OOU zF;uzij0YgAW9-9Gq(`&wD-ZQzwvOKRc10G?!+>O{sNINqCX#>q9qXNF4JksEKf2sf z0Va#05Rl^lklpRmjd6Q?9N=%Qa0+LOqUc>=4uZPq1zpKE_5+6YmGNmI^O)z!$KL2A z68W=%4(v9)EF^Wxw;|R_>I9n<_l;Hi@CeIG87kHzQ|M@ z#Y;&q7 zl~v3a&GOGWv$wN2m{_0q!=MM6$71a%$!c0(^4`UkeGKKppxb$<(2iR0Pt~}I#CG6n z8^`*d%Q5DXgjuM9;`ooa2qhW3E1fONhaaR%^47|9RzW4z=WIeF95Tqy%9JS1Qu)Id zgX!};*(x$FrZM{&ER#(UN1LNolp&g_8G%qqMYAsB zh{N31fQ!ySxN#3lnLs3k#i1TB+u;7ZJM{e>!c25m_3(r;gF}yCaEor^R+ti8#{1Ij z6e*Y7f65=?jzl_B8NyTk1tpJN(3X#7qER#3;^m^Zz# z5-lOEK?n+AhT~7L1-j2$Pd$9*Z$b zE}MPUn?SbgSY(h|MD$@EL39*AZQSpP=B2l%%2~jne$IkO}h(@e$W=)8HcIf_60J=JAqXy^shnEYqway`w{D$J8PQ#SV3& z6tozfC&F)xq~5%)Qjs8f0xsYm1_itUWdd8D>T7ZMjPSIRZj@O9Bfu#EwwyK9?QoL+ zpgYpYE2uL?BmAf)#gx3+3LJm({aK7TW;7kE@6jcUInoJBDr8NUlZ36?tL$Z ziIzlkxbo)^CTCW8ze|cZp(PLK2xd|F%I@spjU) zXTRL{n_;zq(%@Hi`;8b(8}43IYd^*xt;}>u?h06#uhx)Ppn~&R#<$7cwl^Y`Eos-n zTQg^FPdQY2>BrN2SkHuYkMX_XI{Fh2xxmIg^0ZC2-4N!m)cF*j)J^PWAw3i)_OTM> zoy#Kj%xkOMvqksf`Uie9fD}j0f8Eh#{eryfc@g7_Yc}^IqgFo}9nBir7~uo@u%mwF zvK4jWTx;|TJ=w>*O?x4D1`XVPR73*3jg8Z_ZvUavjC@fo#g(MYMKH)0&ADlL*Y_Pf zMjU@AAlS4?4^>Sc++4jjW<1BJvsve8)dJ9ZVCM5HcOVX$4`<&S#yf2>6&C2`dq-Jh z5q&g-)Pjk+33AY>Q-d7t8wBX5CA#y@-q+qfJTY)AyO_Dr7d`&kiD9mhnCGRFMFE0? zzEel&(s4czCT%D>UbRW}2BJDfQcnrN(#*8O?zZf{^GybV8&8ZErM)M=E#a8A3Zvlj*qN;ZxEzWtFTC?UGqN4GeNS? z3@z~)W#H7|Q7WM?H^V(UJ!UMy*I8?g8IzM21CKjWVI-q)?**66EH=F#yYz9WU@to* zW$FEL32%a;Q8lFG+RsprjH*MmlMbWX+Zt@Y_>&JxT-A^g+*Ns_oFo{0a}?sDP1`Ip zFmOrtMI(=wo!T3iZ6Aag3Bw(<2m2GuUExuTtd10js{_0c$FpJUk<4cAu$sHH zeTP=Y=fH{_sdM#GNLo@&7;BQYADNzhrlU8$9fILh1VVVV_<%c|MK$F)COm~ZK_a1| z2LsXgu(gANJY4p%y2M{U%F7Uo5vEz8C80RWtUxC$M@}Fv(d47kyb;@kgcYE+4W2u6 z%nq?T5=h2*5`fJq$_)ZuL8ATr{q5-mN-uL;vz)?EE$I@yx@k~UPQ-;PuIfQl2NLw_ zK3r*#U4VXw(;srf@BO?gLRisqDxMXbE~GsvM~))tsCb;?_yfI1TGF1$rC*Rmj8~vR znTJa+eRKRi>5)DH;>r{7a1$eJus~u09gC+OPM&QPUCKuy9ECish%UBpB-Oh^o7);M zg2xnG%z6^dPI0w+vcazt3X*)|bA~IpX!11H;9ZM0e>c9IGPYc>&K$Wv>cw)- zEdrd&$%tJ0aeIs8$JWuQnT_QFxy?;jXRS#|ZTW^hcmQ=iBZ1pM_=lhD0M)si2p55G z(Y8ySv7K8>JPfJVv|#-1xmAGajPXRk$ZU<~YxZA-BhSMXTp?{1irSVK(zCxQqL`tY z;BpzYP^9Ua+w*-#RlbvqKsKUdU~{d{6%odOqlREq27La&PTu)e1jViwW7$d}?! z1SLn^mjb^#a~V}X_2H`HJqJm0($GMhK~OKS*~X-a2H*|}k2>2<4+QeFd8%J%TQ_zy zs59S9(Cwe(QWDi!(4nQ$8N{=%9|Ac#==63wyUeAVPi*}%x@s~>b8%|Wmg03Mn-yAP zhCSc-BNizgY!gmzx3EYaq+!8dew%8q8fOT4=;#Q(q`x77(cSGiR$%}u&w*UNmnWJv zZRSl_xNFX;U!AYZW|`}lc}6T2I$_40!08@^p2-p7UbpGmFg~p^q^CTVrUL@qoj;;0 z{B@D*2F|j|3ka$iF}H$dS6~HalMq3tPp>uD`pZgKcj>AkpPGm-N+`d8=-6-kD9~Q$ zB^YNwAW05WX9GEdZCQ-l>kf8IE2Ie!O;4of{0p4YtcwFv)@bQG#^iiQYo&rKE0qCW zkzK8CI{71oxA$lQEo2bGoDBknJJrt(IZ*%YLJ+9kK-c6aGu!WHf=72y$iE&Hrre)31KIm}iOvZ&pt3aUp;JNO8$3s`o`t&Ggs>`ncoJKZy z#s*bi|NEis>}PYkZrXD04z_00P8N9+@mk-hJ8x-js)+|4NY~5{=ahcdWKHan!lMv- z#DV(b6=z&}pPu*cCBl7=Q_wNT$x*T${Q2O|i?e)ReR+;YNsKr+Z?P%3Lz@8W=No_{ za(UEry+1w=xEqCFxY)XE^h{W4cHsT{_YL%%oSdL{&#&lUZAG7#l}7gC<#_x&ud5a* zwR403b(ZQd{(|uAUNS1Ow>*H_z6V&;A-f*{bkpy6LO%3EOC_&Oa&&{o`Wu2mLJB8) z5^Z{3=k25|kN6A(KD4y(SWx*A6Zc#Ize^lP`>L&g{Sk6s`WyB$fU5NiE$!o~LfIss z#BqDUimkV7Lb`8t75e}??uF}T6rHK`gK=uNnR%73ZNoWICdY9xS36h!K@n%`pC`Zm z&){&!2_hG@6|2a&EU1(&XpIW74~glG{~gaS5rbDOTHF$gAl|@&Wk<3 zUG$&EIOnG;d={%CxNJ1-7bVT&_GJ+M)%ScxcYg%mjjuaM^e z;ub_rGuyogFc%Mn`{t}#>k91jYsLNpz~;4Vj|<@Y8N`gg#!^k(3-{GTh&Q1*)nS3h zfl9y^8*}X-umdYoQT631G--8;M$3waBpJ<)`E2!yD_a`~F=rSSiMI$3 zKl%1@EAty;AgsSo$1#K(xS1ff%1VU)TDjyC3Aim-uz1yn9RKC3K0J88?9&-+Z~a`Y zgFj8u**j-?sQc$R(L+yv{&@DqQf7?o_BPVV) z#yRtG7<81Zz?YptJkyq02^5_x;oQ1Pv}&p{N;Vw($=|MJ7;M<})~U&6*W2AqTRxp> zja{lXa5e~QwZKv%a)apE_6>A!glHc%-P~xX_GBJl5-r)Wt7oCZL)Of42tTpr<5_Oe zH^)mOM*1?KYDs$JX7l5f8*U~g={qJRCgQKm(_iqJTzqwr0nW?EMnXx0nZb7He==Zi zM%XosG&Aqt)&pC6ZAlgYl85h|@J0r#Ij<3G83_O!hU{(&v{Hl{tZ9mhO9!|R(Ai;v zjgj}H;LZMQCD0l{LjYsjd?prup?Xtg=5Ijb1bJ({ z58zxj-@I`nS7-Ku-1*CGa;t)w|5kDU0F&>NAYlNOcjsuwy=_PJ$^Vy@``o#CcDv{(jNTn}d?#Tp4j(lcx zU*!Q?s&uzK0fx{wcLlPN4fKf0=0u127ydb796)G2Q~30&aV?7=C7zCY4VW(ai+e?z)0Q9mW2IfDYTC)2!dVX%`pObBAg9~i`%FJlR{f|VOU(feNcD|F( z^*bOyOXQ+23FqCurS5;qGh0>zZRpQhwNv}oEL(vGkj1@w82FnFoO3VD)a&*j9p@YU zu)R)yo;I64Kc*f?17G_sP8S4vd*h#uS1Kb%IpILrP%zM_*{A$k=yQOg(9ss+GOe!G zbcdYlb#-Cc7WDo`xPenv;|?~h2y>nS+7lJwqvQ$K%%lvrs+*Yvu{J~q%EP)x2fi&) z50~J9c&glL^uxOs0`@36vJ57EvD=m%el*bVj5|?PIC8!wNNHnl_{@loHY3I;<>4(G z^PG{FY+N+S@^6jr-72n|8a~Ch*Gx+p1-`1`4!HUm(9wOo`b6tJy!cIJOhrCyGT(!| z$tSFBi?-MMfVLT!5$vn%-7mGM$R`|T5LBtYiq!D22K_vs4H*x#QE!iVt7A0OTJz!g zJOlpD0rBn9V=ei{mxSkQ6~S}AU*wzpx| zy(JF!nSBRk*qrMnz@y(U|JXM^t{TNSs`|~<{kcfFbX3I|eWdM$g@50RumDH((8Fu8 z@M3R#((Dhm2;u$I{D?PD^&0Vi#DP4IPM|04F4B$l1Xz673y<3VB@HIS(V=skO3yfb z94q2j3*mu*VDe3#gIn#A)E;^SC#@`JWrRU%B2;e09>E7G&$#_zfKWhnE+{^j>$Uw- zdQZ9aU3-M~;lpwh_zMXO5*0eVJpz_G4;fyfTV^t;D)M32;8EaUcR+p@wFpf#stuT$m$F2fG z)S_BWq+p@b{rdV&`$WR!BIO-Rl%+=ItC5Y10v&56_q@}#7<`iIlfpi3vmC%K{K*ph zt8Msi`o?cVA8M2=m=C|W`}osePDTNR|16rFu&@%NL6dvG&#=0Bbq~nlqVv(Z(t{Ttk40uN^7{U3~X?*0V!fhacyxHz)%-3d|Bu2WZp7 zQ;&S79)*j_n()M0o|)jfarRIv3Lw`V+SfwR5@U>|50aN!wKqSM6m+~!#W8kNG><>j zCvN}>6^=fLPPa$gS?xc|lGcM3UQ5l4q$t@RGtWS#n$aU5kH)}B!s40^pxl$HeGeda z0{9Ax{MaTDknWQv7qil}eF2?IpO6IjAcY*It=o6R>dcOx8MtoVj+#!#Owf*%N!um+ zrrNmEZCc}}t=iy$77YtF(9wZam|v`)*8#L#j7hbxgCshtCuy}t$}%I9mZTmaK!GLa zXN6{_!6|`o?=*9rv$lr1pzp9wMlse{j9MXhWF8X~*jte;D}2~JVCq?V--?bV961fV zm~+hg(l9;?&l8Rc0zl#yIiO1qKHTGP5A4|H8R2>do1SVA*>_%B77D9wrQIwSW&s)O zuBG=siZO+X`f-Qy!v>JwhHlnn;Ej+_l18meSmVjR7v?%vHNOY0>ViM|*vj2JiqlN< z8797p7Fw-L$kbM%W-z7aGPXQx?_V0Dw KQ@qPB?*9N&HHwq~ literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_3.png b/end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_3.png new file mode 100644 index 0000000000000000000000000000000000000000..3b860a887ae3a075c905ae04b444d97af199a9df GIT binary patch literal 13069 zcmeHuXH=70w{FC3*`lDIARB1{DkvpxK@kvQp@=jQ>5%A_NGH@Ngd#SSDn(ErfV3zA zL3#^8QCcJjkrHA+HpB=?0FwX)5;$+vUCwtrcZ@s6ckUhI-tP~_jQ3q@uDRxX)?Cjs zbMKs$>9#Gqw}3#PZDwbT&VxWAjv&yw;LRI>Z+@p;tpR}!zcVvBdEpwAMUAeM3yoa+ zY%x%@{jic%2|DE|eb?zwgXDGtMJk9hgjp z_$v%ZEx(svWKl8=`4i$EoHTq8{%)id3h&k4-ab8snybA@eHVty<4>uO>RT7eLkJr6 zMKHmP12V+q4&mA(tCG&OrMxv>-gKN=#V^0UDYRsHI8=p94{#F^5y}ES9a5i$A%OA6 zT{1pb1=EQ&-h-vw#=c0+74;}?p@JoBm7PfvkD9+(J+j3 zwT|+(9#eZhHd^4ba_DL=T0xtc$mpzU|}=>D)Xq z{KClPC3_VJ2bZm{_Jz44; zUuBJ#SY{t3cmFE3?0k{dJvLnDM;RCwP{-ku?tRb3<1#Lnxg%g77pQYlIx8+iVPtyE zQufds?WtBY+WzR4^Xjt40xrI~)(CFr$Q_JYaY}tT(Zr~zE_5AzCzeS^Es{M*wZqWt zZJEkWLy*-;NC`OVb3D05z{Sl)f?e)lBWP;V{iX`NNqgtUaQcF|wN(V>VJh5avDS$D z_Ov*hQRMAJTj*{;L~*jo(jcc~cGFzLWRh<~4Z-Nb;1x@mIe1!O)3SWZ4VNUYcl5sY zL#u$t&sPQB4>ZQ?Z?V%bm1CPcz!8!orj)&gE8NI1z*LQuoW&^8GVXZjM=a^=bB$<) zu&I4)f}$Zda)fYG3cPK7PrxvBrxW5QZlN9N1W<;Zo;5=}ulYv$E^q zbP==mu@on#-aR^q(y)1dZ8DK;^8(ms#sfmBehH->q429~oVv6o>+lY{S|?Hmifmuu z`p95>sN6-rz0|P!6Fv}Jxi~kTmzdU`5msfofB0Niwoym9<8viyIHTOl)6?@+$nPd-6V5HM9!@rgHamQp2zcZR|dQ`z+2Dvck;IRa6 zfaj6Mqy{#!N2iYTunAs?(SqS+U%uV_2 z3af`?z$|s_$x&-43C}0s6S7RP49h%oE06BllR?WCi+kV?)iEubG!FdY-twk8P90z7 z@(8gb`KX4BroPl+*HKJYW-GVYm%JP%VJ=$6_i8)5tjF$V(6kB{XQ|nVY0A-uak2*K z3sCKG=y=*Gc{PeMRdX~fRJm_oa>~+(<20vd`iky(FejCi#Fp+-Sf}A)sP8(b@W!Lu zVcE4R%02h|nJ!|p>t)aJ*w7(!^wOQqx=L8lo=o_CTIUy;B@uJt28yJnToO*xD0sR! zGwIX)srA2M`uQY?z}7?WQs!yHfF`=8f;{=j!tcMwgNgFWvFz7BXf)foi;?}3{RaN5XIGgX}mTllQ@I+ z<5m|K!yY8FOAbbPs4~BU%mA-WIen!2?dx0Xjk8m&EISAbMuGyu9=|bG7C2Il_XG;j zIyL0izjE3dKSg(;8yq<5JMy;Ck3J}B;H^S=@_c}<(NxD8UlWX*T6&Nt!lTRbYW29E z!+$n4as;SWBbrNY_}yfPtf~q#`FBr%2XHy;Oxs&xDCh@1e;g1i{4WrC9eQQV{S@ zEujdj)t*Rc%}EZ^r+#UHn{*ty<%rYjCWVMfEr@PaeiS<`&%J%>$sO6+ROQXfFM6Zj zc8=~33muAyxNLE!8~QY)gdMTQfvl07k_vcI-PUR{S9MtjQ3a=fVaaKAq*6UXS@PN& z55u(D;y19x9QZ@#LXXaXdqYhga_iyp(&{m8<5X5Ye`0xMwizs=iH->+!!zNNaYWE4~_FeT1k~mejFX+`{&)BET zBSXA5pH?jH5cD#==J0E)zNVc-JP|ce#hJM+F~xp(#Aj&4#(+eBu=Gs*S&sbxLc6D5 zULOsQWeA=}4b8lzr-t=-c|XEP*y~vgI)OvSWaKHQ&=oE-j=1npdLR`!`W6zYQQ4z} zXyw;Kx!ijKll0l!gxqLJg9~8OEo?5&y=B+_s=(RWhJVykkuHl33@kqCRe-=GIiZh4 zS8v7+pC7^2G7iX)n{C5ioz@fI=eM`L34L~EXumx=G@sNHd(7I}T(TE`Cd%2MN>%RR zeS8qCHOLoZ7n%Vi|Lc@ggX40!YrJ&^k{OUhHxHboLFWCJMse~^D!iBGM;C*Z9pY~L zPrb{3)BI`o-V_Apd5u@pHJhm}EK^mo9|_ivoHib{NxF6B@j+GzVLwIE#WNk(p#;y9 zHqc827u=Q)ebe{dVJ2}BVFGr}Vwa=batp7}E9jQ%9U5dkoZ#_?x59cdLFq)T=uz&) zp?Fmk;+mf*PVl7@LX5i{7Tsm_K*DJgNB+e~f=kRkH77l(#;ESr?giBoPE6>{`-$-( z$f`|yQ3s6?SL8}C-g|>29*d!`MolVDwQlX3+xK&`>|*OIc_iMB7A_~PiF$pOQk==K zG(sPJK2x?{SL8f*cSxbD_L>sUtM*)qfrp~hT2JKkD`rh8@i5~M`O87_hfnkTITc#$ zGd+f-f(v)wW@_`$@Bwamb%f{>8M191Av+T`3%I90z)L(#<1l2p#=X!ly7Xk%;qJ+f zNm2E)akjd&gL7d5bQ9m>ee14gnSBX$jc!X-X`UD?C%1;#QreXgOwR2&Odls*e%$xW zRPVOLb7UA(Q1v~P<1%obxtE>5xE7t3y4wOJ#tCGcyKd2^mtSUlVg}U&g3PL}Yx@xI z3xdat`<}MD&z;dp4N@JA3dJSJo~q-3&#pRe7rLR<9(AdBR*|+e6*B5i(9ihl0w)vc zE3li;0GF>mSjgJL+hMF2O>321tpA#I?M$d^e?-}2k?2Llu_9qu`}I&)7<4q{13FFN~XvAzLM*%QvG;5?eHo8^#(!>pZ=PV)#?e*!;+@QC=pzjH`#^PeT)u zw{OUR65bM0cI)C4Nj)1)hzvsG0)Im@PrOzvP4p&>-!PWxlO6tecw311YS&FMwPvj8 zDR<>Z7*snC`_QMkYeR=!$H3PFd0iU0Z8`A)fHluaGh_KtG}!3bOsbXIjCyeYBx5VK zdW?Bl!wnXsXAaE^Nyp{LORzQeh57dP|9UB}*+1KUn^aKIi4Ms~A4~i{av@m`Msx(p zl5AsCJpPIRb|JSgBChld;pB0DWWhlti&(1P zA_Y&bZD*8a>dkFS_IqqKtZ}(GPgIwk@L95;Z2<9Lgfn8y-Na*$7p&00(c~~9KlhfK z7|#Ff(T7yAXN1@@tw=SJlwj=s+i`B{bdR0}3I=}Kd8zXBlgaeFAM7Plp~!a3mIFoWB&q551FBDy)m*~a5hygba~8hYW1g3olyuCO=J zYfI4{_3+bUqJq@BzVq=T^a@IeUHfg<=zWuMR_nc0-eG+zxfi0zI;snIgIgUnP#U3z zTH$f8q4^b3$LMqz-N1_+<`nd~M)ubf)~R}ygB-liCzjEl#TUzSwT5G=mpb+S?gFge z<4qci%(|0Sp*{1@qf$VZr2sdKKT73%sM?WYs}l72@#aNvg>MSVoN_QRsw#r zwC~Z{xe?lZ975nALT#Nc!sq1RuQlG4KRe`8!weo>fe5Cp^|7^|D4sm=QvUd_();Oa z#a(I2KP}f%`0Qb?ody>)(Kb+Y=pLzG2!{?_fIEk63rar+`M?`?d2RK~uaNL?&HFjp z{+-jug3e)Ss0WVRxhsVf8uX$J>0GsL?)L27uBS~(i;avrpy1T?R+O13fSEfiYnZLv z-VP2g&#K9Gwcf9Pj2$%`s;Dn_*bsL80d5?K6t0XDdo_cBbJtfQ6mYug?a)g_aQXuk z0(eeRw^tdWIrEem#tIN9n)A#yc&*nxSc*Y&NaU5va%1sToud=qN}}Pqw>KkZr+8Bo z_W7R(9`?|}**#r-))ymgt5!}Wo?DbT&F<)33-;ultwZlu5Ok!gI*U;!Q@3Huxg2g- z&ni8C0J}ERXEh=$h2BSyd68Z>`4d;ko4cNZ^wGW`HYz1_bnhCzlx?aggidD%ZnM_f zy!hJNMyjZOd2YwR6xOVY6sJdU<$lovqFHBUM5*aTW|!Xc;b-an2$8cfHm?k?QjhkE zgX3$;1!4bqJe(>iAZAJl9>1&V60=#|e44z1BYKoDuU2aF5E+92TIjj$&}DnlJ<=Ss z%Phz>$OgVDCen&JL9q6iOAfsT?veM29sYX2LMf-j2UH{k;8XVmt`{S2saD>~QOPi{ zCW}5WIKsP=kTXT=^etlk_mc zm{q+_v3k9Q+JUA|gJV+Xk97}Up|$6uirK8H0(z==VuT~yX-c*!Xa94VX+1wBbe@FY zGUHVku426+;h>D`qgdJVnTXgS!4Yr$a^JG4(4Iq&Jix~0Sc2;!;2D$~!Z^8>4Rgta z#mqjaxd**z`^s?1ar8ngy>Lz{4E!Q}tjO&?oQ~^^3CIue*c7EwOGX(O}z5S;f?~6%|yE zO;-x2u>m)3ip|QGG9XCpo?2GXO;yG~7e+Jqh%95p;nD8eW1y_lF&h$;4gB_`C_SYh zavr3JLG_{pme2MZJ7+|xF6V{s)}>DI5)Hv@(Mf95*ghpntKGT+favH*aowqiC+!_~ zKDc6lfp*tZm@53wC3Q8=enlyQmsK(i|*xTs~s*v&tL1ODY;x3zSul|cUk`oUm z)_Yy+>&R6iDdo~*ZNnp00<2TK*0rtQu^T{WN& zF(RT*F2+CcpMD(=tj)(}K7}9;PE!q2A#_+Cl)PeZ@u;xhr0~KRyt4o-PiO>3YugrCAo)V5H zHn^M!H3#?TVfX?1maD5IYY2L5q2g*Wgx?OaaDaxv`zDzzb1+UD9q{t!>3!fZv!;b1 z7yZf7k0}mEUEibTaMR_8NS~}DKK6QxBbP~`!*0XUOI^yzMIdCr4|*|$n!$2*^A$-T^L2VDcE5hN>C>mtmqI_fEEs+|ssI{Pt-QXa?)ppyG(FU+4%9;!Uwa#Cm>!xsux_sfmf)n&6#qMb2Hx*0SVR6F?HVfhKrnv~Z{xsPmkF}MMyff1c z=@d?(jAbC_C>~ z-D%1?K{aQsAY`7xu|vx$8qme6OgPC4x;&6MYS!`UYs{1m`@?*kTlmmX*u5Yr(pR!`d z=~9!?99vzi$ABXyM5kjR{7cBJ0}$3TfaEI!RZn?ZuR1G%zi#XW+#-}xHM4-s#!L8| zvKklg^47w_0cuQtlJTZ~CJN01{gNn55>X&YjGgG|3Yw;d{*AN@=JM{wBo&BNk~2U( z!MZCLDS@W)VeZzGY)&QNErZWt9%oOz{IoKMhcpk-Z-{FC@+%89zB0zVG1olTihAXx z5>^3ROb@Xr^?L$n*{hT z;KS-#ni!ueiKFR|k~Znm_M6sC6*0v7Tcb11R0G3`N&)Z=-q)F-QZtugT!>vhM!@1@Ob`lT69(&ejgAJ3EGTI|HpNM|9NHN&lxorV;X-guEu4_SY8Ty)^Id`?2f#*vL0G12xH=e}4oS{y*o3W&4ZN zs1=siKv7CHtlr-(%K(MUI_l?~1&NMw1_;}~Wdrai8YnRfx0Kx$l`46l=1>BZ5tx1E z8ATgJcX31i4A4UGh74Jp-oiUuQ%T35A}@e_CbJS{bV?e7>5KpTSkq*ZzKNux)|EGB zfHIiO!l9N*rb!s!);gSkE&4o_EEj*jpNvSBN=}go+e3qtG&l z`!sJ-sCX6C`V9Z$A4)Ff{z%}+ot(#Xn=@ls$b z;FvNOo}lKLnwIgqG-m#qxwB!HBLHq@)mM7Y+G4DFTLmikOM`uXf=+VlE=N$$ccA*; ziyRS|F{ID#CJww%?}Z6Fn|7ZTlV{^6`wgpOwh6ejLcKN zxAFzEDw-D*qrqijC8B3#$?qXlhS-XNoP zh(PN6lz(D*2+`)h3<$v1zX$xc4~1c4v*dg1KJm*;g^k>$ohibw7N@lC5UJU*!Tw?$ zCw}{v=YBGZGYRpRB^-7;$$zN?{&zyg=HiAc`uNQ;3Y5AX1-_epQJlX-Ir2}jXG9h| zbm~KUozE?hpHYnDLl$4@rzl^F$=xcsi?`Zh{f2?K?83DH@_zHc^G^9}F$3<~ke5HJ z6aj+d)&Hh4*@*scM$udTR7kSap{5JZRQoc=6f3>yB4Wc2Vex(bpE+Yww?1L}mrB-d zU`>ruj$zG6*4bD&q$9ocztrVApZ?6gbQVbYCqv(6lg)2U0NqSjB_!$3HV_5UT33o5 zKYq+y9K)eV+-cH^p8s26f=l4jLgv3N-B0!pR{>&4Z3K@kY#Jd74%n82_=}5+V}Rmj zDfF9xs!=M_AIJazz$rR^`0-u62m?^?&q%ojytsQw%Y$eE+_n4s3H|$Zm*4dT4rOS# z3;@FErv1X5b357`ldzo<)h|Sh5VCN7Kgi*_5oDqec(o;;?rZ`0wN%@i^qgQT`}wKf zY#p#4ptNfBel767nVwEPVbYZ^jDa2i#{Sy2)I6_iob2%&w-5dr(3WEWuB)@&H){P= z@qgVaBl0%)TU&R3jS`B<<_O{3`H4V5$6?QJK9%{yr{AHkfVuJTsf~%k;1*)w z_nrCoL`UQh+rBcf2}ru-L(_W18n zc7H2`YdB!=e~$VeJK;NYtNv(}f8^f37y0j<{89e>J6!+2$jNOHF}42GCLx8b?KfJZ z9lgr0319Hm?{2FCioPErM}M1?qo|aog_=W!1<*2PZJwcSCb~<#GHt6mP*DCD-+*cM znPThwsae#qhQP~N`d2F+{T3iRw|`(-Vsjeqy&>!G{2>=^a44haV^*X@u z3tZMOmjE2oFEC)TADl;BRzNrEp^A;IZO+wKEQRVpmR_j$V&W;D8_>UceLvs=piy=E zwjZ^q-p>u6c^Wd6VmkWb!P=RXissp~o$@GOJ%F(%{j8F6e+Uc9Lro#R-P!hz{bpI} zY;DJIkSss_sFML(TMiky6kDv*GO^98{mwS!j>{2}<%Ol8%>eNbvOJCPz)OGeh+g#Y zTAHPz12>7uo}r_!LhJl#gwt2w2by}wb~x1cIgydg-U53+@ej#kh#M%N8_O*PmE3E;IoEpI zjRJf|)8`Hu(o;;X-fzb{9QyT2o$I5QG0|C?&CIH=M30cf5Rd-Sjt-Q9MFcj@zZ9(w z9Co|vX7B!3+AgoUl!Y8zE+3U|8~;>$^Jm%;pvjW}X$bn|Bx^yzi?GRQv5M`Ebw0Y; zm}9KP44%Glns?3hb&nm&vZ+fd9b&~pxdp`KG~xW^)9}M*JR!Y6kuL`|T>n{%B&(|QvTbYZ15MmNiWhxtJ&L zpe~VX4Gz0TeW?F5s=I@TzN5s%mdXiv)e#;T+O zX@t)KoX4_GXf8uv&1x*9iR3pm+1wo@TG zqRzB@_)fr2&!p`bPrRu;HV2$ygNi22Vaw-{2hVo zCkhl@I4rims6gDiKDtPwov2IC@m-&*!l^nL+#?*imI20#aWHwzTE$-=NWs*w&0te6 z>eJ)c0YC%%ofrx9KT8cf$IsseGZWgs%+P{|{3zbTSXGLA8*BkE+=hFyeA#rA9XhUq z=k=oF4-h1Jr6w`n26pA+$D75h^;tC`KrCMwH%c||2~@=#dwR41XaCy&qXP z4_r4G-`4caTXvN7UkXD3mj3^6^ZK6@@ekVLH@>{HO=ci_j9a1|xy3p+`{+D>nx0DE;~V+jpDL^c2clg;A=@QKuke~ zgs-iP=gD9D5F_k;4*{BpF~UZ8bf5=1)V^t^^w|K=mQQLP7tH=OJfG8N4X`?Iwj4W9 z1+wt{I<7mV&Tl$$4h0ME^?|1Cx6aK;(*5`_WDd84?7(x2!IobcDtcbDpUcZ-E4!R!00W>d+5G5S|tr&d?ztW7)mOPpT zC_JGZ-F^q+UbRN9oE6QlnkevIT?||VUW*mJ7Vv6^X+pzB^NM?^jH-7jrt(5sYrHW} zW!R*f`x~jmvUtNea)8Z|=#^tAg|=7i_Mo;C_enRU+(K-^x&5Zq%8pXdYnK+Qx6Odp z*kDrR5;fC}zbgD00d6)F%Sdnmq@tr1niiP|`3%Vu@OI-tK5LR03EC;{vJSNVWD*n& z^x~6sHi-jwWbROX>k&V4Rk7A8fZ%7N*(%YgSKd5K`Pi42Cm%8Yf_SWPbtXZJ*wwnO z0qEu*5K)cZa!3KNO&{`yZMROZ`ze2XqzCjc0Qk)YSH?veI6MEZ{D@;E+Q`E23FxE4 REMNi1%-G7P;?%_({{bPkcP#(_ literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_5.png b/end2end/liger_examples/medusa/docs/images/Memory_Stage2_num_head_5.png new file mode 100644 index 0000000000000000000000000000000000000000..f7920168225fa4351cf190cd71c78844ae4ff77b GIT binary patch literal 13336 zcmeHuX;_n2*KQmTl`2~6REo4xMMVZDKmwsCO05D#1O#M~$rwUZNHBpUimlWEr~`u_ zpbSA762?RVAtDM&7=$ncB8iF+LK09infZI-+kS1&`<`>XKfZIF?_6Jh@Z<{D^E`X+ zwb#1W+V{P3p`Gh+(T}z$Igd{ z#_-iAkXYT>&AYe$bp4mM`dvG#uJ(aeSrJ$#2aD9+9qZu@nl)3DZ3AZTtv#&pfiE8N zGjei<-rB_auBfQb^eZ{unW6pe{XXUlAo-x%@RO%O-8Dvmq>^^CwyZYl^i({brq0i9rj`*6*NfWH0^l4xb= zSNVOR>J_gauNlVk2brt=Pk37~Vn-U|W2joCfY||&)l4^jd}EAvz|P$&Jfi!Uq00fx zwrhe~jB7R{#uQ@hu;M)K5xm45f7_ZmRdm_}J?QISYEhVcl;pVM^2p^b@<<4ItSQke z$$ID#E4nIdFv8@iPEn}q3ytR~zE)H4D#}**eRy#d-p8i5;jndMWC>$em!>MCOjMG` zk|?F`u^CQt-%~%ny;kO&;nZFa@e;Qh<7}}!F!#Owj?)()PJxFx;MHrM8t+`fVrZj2zJk59U-IuRvc0g9_JF8I9TrM8{lyM z{p;Hb)CgfX@xe!J{EMq?gZ`mabtl@7m9e==N`4vBpROD{O*MUS6fgD1XF{|eGW>^4 zOX(l4(H~^J@hMau!+d;`T4DY|woOOZspv^jomrchs?V=j)mgE5E;_~ZgO)0jCL0O| z1KTq;Blz}eb5C^Ah<4mXTdYiit4yBS!3-?xmR;$FVPH>!hR=Q0#<^KdaR7{p2a84< zDwM<4?1a)!ruDYzwWSo2U{P!>cItjAp^IX^$OA2&vp^Vw@L)sEdNjH>Idg#;ar^9|Di^v#`ap=(!-4 zoGmH@cD8~p9|ru%(I_Hp$Y@BbQB|Pn(jfYtC@Kw87r$kwGQg)hh=xH@`Gmawiwg`1 zCwgDS_c{+&oJy+bBf>~Sx=DmKdKB&L!3gI?vMgV575Usl_OH&g-A2Q;6QSBtYJT#7 z$TC#%19F65sma5Pu-37#b*fVg1OlcW<7$G(u?_J*2wSl4%*`J5^nC=ZHQpW%TeSXG z^LFXLvbyC$6sc*3db+Xl%qWXhPhi7uk&<}%iGDh*hO5(j+EMP=4$~?Mcxoq)OEKS0 zenr;8-PLJ`DO=+&fLE0c0Dl6Y45>Ytfy)-YH0nKoY`nh$I z4M){XP>?VR`)P#UsMn10;>MZpRuz79Yltm&x|(U^r-47VQ{V=C?4QSPW;Z1w_VlK! zN2H>u`&DuR+f=e+q#ehNV{WkDm6^@p>?0S?qJm`PVrQbjeNSE%9g1m5cIYj1ouEfW z#8IF>b3%S)qc160Uq)A^25>j9?TyLFGP#yts67-D{#h5Mb+jJpv!{2%t2?4IlICCU zGOU|oJD0yGID*Ol3a+4jC41LJdL_CN4eoM1f#H`9&b~_IVh1O5>Rw%?HGXne;SHFl z4zRb~;zErurp&788S062*)Q;Kh48hs0{GWLtGw8kJNnRBu{p>v#wmF)DaPW2K!kcR z+&!_cW0!6jxTB1ZN)4EF8f)KAg7KIEX9xMxb33~*3?_WNFcgek!U9y>M`t5QwgbIp z0dyR8Pj^p6_5A2(9iB6cXYCn7-OPEryj#(hm~@_APJX_mKT(!L1` zG`855x#OWpv?^ID=4!_h>KF|T(91&Q^`2xH(C}U%!B6(#{yLZi5(yUte%g^w`!YMbV z5rM37=!)O$(i2Oqif@`3OXLRAUiPugCIKh@fgGGTMnyWp!qAPv9jEYI4yW6FzwlH> z$RGPakD@>k&TM#KO((M!*o+*hs0Paz7G&e*wEYDXWfM8ha4aJ{?s|P#!?O(!q+ZY8 zQM@FpycNEWXmhhSjR)B`QmATqEY5l}`Rs;CSG2g>-k%;rS~opchL6PsYxCgQV)R;N zjZB@DM*F;LZly6B9cg#au8JFUNYm~H;KCeom>3+?6g^F|8vJC`Zud2C!)Re>Q}RTn zzo*`2dOSFHdxU)ijnN1U=Yqnel$z84ga@8+d`C@xgrFp(IkqDVp$iO{>fn4?&yI66 zeDR0%=3q9rbKjjsB&qtZr4^g*sVO==o|WBAwha7XbSB~!nRf(o^QcYVW;0|ZS8JQ` zVzb{-A!J1K*}3`(j}W4vS#q^~cRGe$uN)7|ro%BUPE1~}?HqhG6;zv?C#858CswkD zD8sVu{nj5)yI=Nn*LAdrz~VliB=8q!y8~Nm7sY|sg7Jq6S17i87bIDIxn|i7kx)2z zlU5GJyyig_RbO5+J+SUzi%5C7lNFU0fsP)6d~xNk1l6zhuRSQ-65QB{V?tlDLODzi2`1;~{KEz7mElPC#RXD6kYFHJggE>^?x!%Ylc?9-(7Pf|6XL@%vNK?Kjf)Po#H!%x) zQF(VXh$zeH<@)&J(*;=XOHMI!IRYxhG(QBN(@|1e{Dz!XKjH9-f}i6u+-=fCcado6 zIreWszS}B^3Ga!lGz{Q|uqoc12y@*VGwuDJeP5ef9lvziQ5}%c-79qENVMmf+%Bdt z^BCv*(Que%3X_N9y5970S?iX)--f3uw+Nr~_mTQz6;w@D;^v5=%K?Vu>vkTq`yqQN zd|BQADX(`)t<8zu`naYnmQY7*8F$P0-WP!!lMCs>z*Ir)Lz!b)t80_L-9s0}AmSyr zvWrFVe=NRG7` z_Z`R%$SUdyI8#~+-L^|n3n54`8iH|=C=sc&jd1Q#%2+LY5haR{O9ua9;nv!j>T0<6 zXaoYmLc${=c_5bRy>t~yU~<8!+w(KByk8szra)IHzPf~aXoh~y9JS_|e?xYyVvsHi z%y#sSpR`z)c;`jVgYY3&>k5onY{yHK%0og`WJYM8)6;xIP&DHD=sS5hd=9^@(q0ps zIh;+)kCOMI$}crLpAxR}kTrR@cU;$s!aM3=)SBX@b|n6)I0fvhX?Jn%JNB@IiTYv( zqwq(fO%NMyEWQg3k}Ia7G_q9mk!Zh4(S8@y?k57q0c7JSfyar-K4w+RbDKvT;zNm&9zbxJoSaacbee&i!){;!rn6hlDMKv}n(?)B2Q?1=0 zh)%m4&GJ_EsvbxYY#5CWuaWvUERNWZx&4Obde~K8Lt6QNg>qcG_#SwoCvNlug^rpfy~D9Ium4 z$FQf2Hx^1xgHtyvDXFGRBEP+dUW@iXESNMMdeGC_iI7sTDLc*160--%s-0c&?LFW+ zC?*91Sx?2O+MG70l3nr5Jic=n%19$ViN{5+@UHOn-$lO7G+jXg} zfDKbkwr*g^B^$8iP<}~@-FzL2+=FgWA*0y?PyvnvSM|kW_XXR zq}t|am+?OAeEvCB1tO{R6jO=q-->6P)4ZG5tf-EMxa;d*?GUA9u!-bRc(B6dp-8u- z4vHdP$aDhl3o{!WoG^<%G6w|v`ayk(wJ;(t`oJI&CT~sue9|8$Rk06{@;Wx#6l05# zY-G1B6yrD?vNyS6Z?f_UYVm%yo}Z>rNUOiF2t!sn*) z;tCtZi9IIv;KHtu*Q{}8l2KA$xL9CNDf$2b?1g%5*S&#@YRQJX#|F&72Rcpzz+iQ& z3a>g=nEzOKD7ykq)xeu#Exg&9TJwBG)jOqBGIylx&jB;qf_`3R%sM0@Usrr9S4)?* z75H2qSzQ!1a)}@UUlzKC=z8yq%ttW!m^ll=5$3sf{xRe?aPZQwIWAP?Vpym4exq(! z1su(4X&GFU^@TK7&5ty=tq{HdD3G>KlN?gl-#TGyB`sdjR=EHiI4m*UgI8qZM^xw) zm@&1TpQq;Tw||YJvCU{+SPim>N3MTSxUfE@&rK(*)+ggNCPCN(@VTCQ{G^uVcQ!x3 zuhG6PH%mK})C>eq4U#Q!_S!4;y1z`E(*=G(9N6Ol1Kcw|u5cfdekb(lAgL@n#_`X= z1Eh^ht7nxFk2Y8a|6=9Vx)Y`Si8fd7e_vMK1M!GJz^2&oveQpOoP%CeTig&yDxY!B zF{}o;^=(Z7>~rsmK^|Q9$k+vN3Y>IZ;^rR3fDbq9&X<&Dr>(Y75lxcbzVok^@oNe6 zr~{n^Eb?|4-zq7uq~(n*kV-`A%K8b!PNGSI>AE04wgXQm?tz%*_Fn8$u@919CU#AQ zcHMFDzxMB(+DXxemtio%gI+Ohc9w#;zTtM-AxLN$;4^owLkmK@aLufU8tv*;Nb(WX zv9Oo54nC-Q)=-UAhO>px=?m42xn&h*4$mIP=J1ZRv|#7D(oeD+M!5{GDvI}PaG5fv zi-$s%5}|>4WWjVo(wi{%y6S+fC!O^W3HX&2>=})v1ga z@aI(-$fvt*rWo7{g2+45aT3+RbD6ycHI%(Y&vhO5X$HHt&KZ6+SGL=|`-k}!dlrme zN-^l#2);4G31RRWS$LaW+rj%Hb+d)E^7JR^U$S0^iXoUiYv-2q%PuZTVzj3#SwSSl zJGnA3H|?frL$xk#{^Rra!sgX&p+5_&xj8Tk?ipn*=b+RIaMk3|lBo4R6Lff+4o>cz zS`xZeb@>A>bma3J=>h&1ZvI$(As~_mACd1KvA8?h(hL1j#ar zi@(gC7@R+7b18?RBI4Pdg8+5Xk}&mBNTWYwfWjP?u6gNdbd3I45uA6dkU0D>6gT(6 zf@!0EVpC$Ksxdnps_f5RR8Zy;iW-UHOPF3>F3Mt;+&JKj%V{REAHeeBW&oB(GHFlW zYZcP$kdK*u$e*qt&2%RLNAZ3^s7@)V&8BT1OBQ$2_`OczBN%xTH$Q88z)g;#Cy`(Z zMoA`3ms|a)+4&1m1({M#QRyg}7jEtrKL$4$gJDjKYd_IGA_P9k@yzS*5Tmvfw$x!N z5e%Yw>kV`dFc2JUVf7tS$}m&EHmt}m5rObxdwl$Bv{L+RZRlN;Xf!kK(VNSsdAZ23 zgTd^C`1bWNB?1y1A{uM54@M69LizorG&Gs16~?&a#WhZ3Ny`0c<@VTTS6A~}R5KMg z-YAUZVd+m4c7wnAG-9wE`%>D6#EUSY+PQ2d)5~Qn&#y4Xny#Lc)VZkYpz@iAytHNU zO~DzLaFhl2Obm>s?im2v77i{W50HHP>CLetwe@f|d73wilQ%8sWfoH6#zq}T0=wza ztl*KpOl7iVkU}QFy|usuFev}Jkfng_V`Zo?YWgzZf;Bx0dsUW|J_M@FXj z>dM38DExCnvnWtP@t=G^%E#m@qrjcNCMK)ovu5W**u9uW{V1P`vh-K1zHLdWLkQjL zR(>e-!;`fw)zpw4qmcLWX`RC*{z-rI2_xjq+(q^#;sW{a|{iZ%{ zjJj2v+89hTYuj#iA?}Oq_bz>!AiL)wgr*g*@^acSF$XXjogD8Qv)I|B1-&p;fKB43 zZ4hTsnl}ZXd$Eac!u@zqYUiV&+Taz8iunZLDKE{5jW+*j=^7!Gu8HGz=GIVVaX?bp z>#Y4aPE)kKiW{B?k28Fn@RxDN(vx4EA5R-p>_Kht!WNh|d>kl)_~s8sh&>(N{B&`0 z5V1cy{3V;jWIw1|RXhu9@brm7Bb(+kok*^@Kz__3Sfe(O;p+3Cwgd!8w@{+n3i8$2j#A$WztV^dXS;mVD3pIc zb;ERbQ2i-yJt|4X99k0d$RES1xB8fZtl>0r=L2n;h1BTjdkd(P=iU2a-V z1b3o=@#9;8MIr~h2Ngd?b6c>3H!6i6$1}WuKqcYG#U3aL^tN~)%POdxVzcC%pH}SK zJ+!JBSmP~E$F15{uVr!uA68}kgO{yYGisM9c$$=Y` z&vHB`@>gW1@2+e)4zhLaY1w_Sv9o4k-VbiA^DitZGO*Zp;dAe5P*}>T#w#oCxd9{O zd>t6Kap9JgKx*FnOIF_E-*27R{BPa5RQQq%@UrZXoaL|73*rlP!OMd-fW|!-9zTIL z6@?j~UI*Fah9pex)HOMQ+Vg%JU}yj0Jx{Uw~2fA)=WxsO%?O#$>=jiXV<=gW;u<%Zo_#uy$E^s>RfS?_uG+{`VH$J{;JUY~;e*AUpA|0n+~y79$!Fh)up zro{n_nz=_1C9Vp`0T^M&iQ7I09TBho?9E>uYOO8y2k=->pVDxuuPEX5bg?`JI4-SaY}R8^1PQl`gaaQ1FM$%|X-z=x8%KUMePd;s#%tBnOuWq0ulTP0j2;j+Uz!1#FN7qVFkiT( z?|Y*)%QFCD|K)K2cJz2+yFS5i-8b*=kAL|GY8+WKHt*o#Bg+t%kol;uiZQBeh)_P>iK#UxfA!b`9d5J(Z9vg-?sym(f=6B0Eo;+(49Xq z3qZ?=XIUW*L)B?jNn*QL)~|XdVa@jAzXCOK)(|lm{2Qi6QZHU#@v*NaZN9Cp)F4N$e{r9{7(q}J;CJp z;)`ac?8O(2O^1Pk6UbxJdO9I}?_>@q3fy0h6bw!C(oF0LKAwA{eoGK95*Dog=6!L5 z$?u4MFtFMVK6={|EUM>B9Jscobb;aWV=l7*`CxV8(V73lQ+`!}nMAr)$(aC(@LIX} z{{GbdX0-UAqZfRnc2iX-9T!^!q>s5;ec&XhpLgj8rCyJW?gQ2wd~o07HsH8$y)tYJ zSR?<5kbYvu`AhhIIG0KXiZ5JBdfFl zHazXyo&}tM{wdkCw}j;<%``&0bN4`j);p-aO|-JEr@!L5)xQF_G_3yQ`Ip1#yD!HN z11GtE%K8zZ)oZK*Z!iDK;YUq#;w-2B%$t3U?H~L9Mb^;I`7c@abbN=N1o~IAHN~qx zK!KjR!CurtM*67+g_`G5h*Njq@2=Rl_?yVUN{jRBRVfV#{Qi1?2At-3n=*)!jdh(%qm7!VO!>j-}w65|Ro8tfld00<&ig~>xi1#V(HGpBY z%?2I=9(UW{3NUVG4*uo35A*hlMgJ0uyZ3H^b>^afSuVlo3M}Q-xxMN-NKq@s#~<_dq3CkfpRI}J<;U?&cai%!ZCs>+DcyggNI`o>beQb6)bk_it1 z^3gC;5F!Ueh8<10L*Bzw&9hMTvjoLNc3E%}s+Fsq-I{$^J43EL1g`(6-F}7(ISG z&3$0Cyc6RB(ABeEV3hCyXJftKWZY+SPE2VWbqJ9kXZ#wW z9Bp+wjVW3_X9nf|XFX{}FSqlcTpJm#F97Zvc1)hI3=NrG^p$USq1id5vX1=4$RSjHVB}|T#nCNTr-m%GQ z*`y~MGaX2$YNrL&TG_1vla!HSL0L;WbqGF&m5*f?=+r#9HCrlFwxfi-*qQhwv2D{L zbb%BQ*OYtU>;)eMA>f>K9$o`P-S=$BjH=zfr6zG_6ws3OOvn%P3Kmd2hu`R>$^(7#TpX;&;b`KWFHrb#LQDvhrrjM6%CPdknL5T0!TV~%2IeV(mx-OrDonHkp zu7mx`yF*N5HsG4bhVSjMqOgs7oV}X>w+9N_8tGs>pgMM?6Qu)sPcfaG<<+bz6NR=M zj+x;*P6txB>PdU%wNUlj>BU#D*h|>LqtRe-V)M8ue{V0^Q+nlz`v!Cgu=9TzrDF;J z0tZ0p`|ogFXJicd7=~#x;rzPB`b18FyqQ-D{5k8@4@^Vbxu+{*KcDw83^qS(yBcUk zk5sZcAby$UK$E+ZrX8U^sFa54khIb8`(g9mKN(v1__)LLgJQy0HKRc5F!Ec=Pc7%Y zK)Y_4UVouYSRoGSwPXYb_?p$@Fn(QP?lNc2&xH}8R!8AXk6{se7ONgN^RmJ8UF%mX z#LR}_kRX)@=+*-)Y(D7X@rQeLYB6&y{o}I)oklJYN?PPJ8~rf}suxuBE!fJ(hLD$e zav6p09E_gG)F&oB9FY_RyKy=%jvB|seUd>(fnOB(zOIa%mlG=rS_N7h2}CX5ALxl^ zT84bMswX)U06o17Xv(gz*BRKHcm0Q21gLO%o=zX0a+=MQ_sz;XGtjRu8U|iF#!g@& zRk9=*IZCth!i#o4&|?-_ugB|8tBep?EvMd0-D*~O)uRWHzA$28jO78nY*c0N08Zso zu^MOEXJK>It3kGF(!n#2w(`P8_Ijg2{7_GpSd?f~a=G>JNTW9jTL)zlhUO30jU|k= zXbVCX*`irXLF&9s{H+dCfg#&;@A@@J9CUZ(Nl+e__}Fs}q#t s04PDl?S8;7ZvHEqH2%%Ejp!Ccv(t8(&N3Z=WDPoWz|)PmFD&Ih09F2Fg8%>k literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_3.png b/end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_3.png new file mode 100644 index 0000000000000000000000000000000000000000..68d682b6b8600afe2ec618db91bc47340826d438 GIT binary patch literal 13950 zcmeHuc~q0v*KX7n^k)IB11dwLsHh-R1;dOpPJk#10tr)@2`NJu0!eI95fGwPfihPF zWF8G;hzcRrFb4<_NGdT*A;bVlAduv~*!I_co!0%<{qFazdzb&PRzlwQoU_l~`#k%3 z_KCY}ZK?3DUH<}sKnmy2oxTDBt#tr_)`YB=1HKt!hu45W$IhHTeZnpR#vN6ta7LnK zL2|p^Q12REGSQ?K-JQE^OuD-bTq*UQHjF97Z+pMN$5xwEsCL$BP2#|goafuN!OyhX z4#!_Ouzk()@!R6#+HnI{Hu-FJzqS5OEa?3D^RIb4Q7FblE4#5VTr}P&!U(fO{^Li7 z%qC`r2+V3?HPb9AtlDV}2z0z?G;}R6t1tzH?ZBrayL3RHn_uYz(*^zLQvv+_uE!Pl z*`C~&z^~4ojrkM=x_?h@BM7v6=YR177vU8QL0Nd>V<9|$=%iG^By46sQ!*QloI59F zSx9rE$B$-DL79({*EZfr&`~mde|#QN;#_~ld#G-4EFql>aXA(^;!xHwT{QQD68c#N za^MD=z(XUJUR#=9k+NN+d8Wca50gXbeA6^N*?T3~mx3z>+U>xd7F)}N@xU)pT{UEP>PJLdY)pHa z*2BVvJWZ^!DGhdQ#@ij6LK2B@EoAc+$mGX*7=8k~1uXr#+DvN-Arm6fQPB&z;6ve~ zNc+eYKAAG!kxr3Gc`{*IVT4Y=Gs*=;^Up-m8szI!fk? zt|1FTgmfH%qTaOaaoy|in&_r0LboVD^lR%yjHvK5^Au#^J*3Xl#Rv|X7v&NuYGgvy z(F?YjaL>^Oq*bX$ly&bMo07gVgXxhInY0n6Sip&*KKU`;x98Ei+$!G+e8A8ZUBW`|_=c5ll%jK-Lplsh2uI5) zB=WYPS<7hCxuGr$9)jfzl(LImBz{H{86jz>>iLx<3&3TP;dHzpc|(l44RMN}I;NVq z7_rAZ7N==mX$}n=A-XLKxonIf-6Khw0TGO^+}QS~T$+%;)qV zxfjs6zHyqWJ$24#K|f*2Hr6m!E6k^|(;<9>oP%s5f1fQ~C`gCPq#`0Qnyu~PJJ7Qo z%p*NW7Bx+nb%fRZ^zFtsMc0bM9KBKvyS|v!PWmj-y>D4XfKcE&-<=qhw^)1`JpR4+ zvBVcS^A?>JvK6l=>$&~c;T6N+b&5~AG|;*up;RmO!|au>=0|YGtsXd{K~j97E!ky9 zL30p2!n`e{f!Q$b*Ecbhv)nf&Kn2jX%bTipTJ*e@a)?s@qZV0OmBhH=){wok1x%(% zJj$h-+w#irxw>~0xn|LXg^1&C#cAgILbt!Vqo9&T{wW%^Z)H&5q4HEwiobLzTNc?C zoGtDvNW-$!CYdq`6Eg4qLe}ph>Ay#}9J?P#+-EtpP-nHQ7HDN=Hv+@Mzyd_w${fX8 zM^uSJ2h5NQIg_9#yX2GIca^hrIHJ@)DJ+gR}IA$!tB z*LK`|+`heO59)A(1J#-Hc{r0Rq=c%@;zE!5UEV8+SD9av)xsBaISpZFptOK4>IqRH zl_PkTSfj$pTqZ6bzipkOCPLf|_hv42hI^TI3Tt02n9EtwIMs2vo|P*d z5N1iWT$z8FAhB&GBzxI8M4~5~(+fw@962hM*s<0gpQc$L=Zvqvd`Tv(C5}4b%Z}pF zI$L5noLb@-G;|xU&cm^3u~j@Ko%Xu@`R7`4Z3wec<8X1-;MMNhcMk~bO&BZ zrC@*+z#9~i77R}+BeuH_MFi523&kl#l#b+}kqA~If&LuVye_xGw_@zp&_S4;yQ8W* z4cKTHDIvU_T3$*+#E?t~AFL_JbZ(4X&s=PSc0(ggXp#fEgP%hTx3(#mMw zTfho!bCS-2oi%0keUPNqzP8=441wl1!>?Ggt(SDJHu0#mt+2co$&GI)8_Ft(nY~&-<8aY1RjWd(YyvASfDmR8!=j@} z4V^i*>;Bit5VHl$uRAh%af&hh_@u$hY|e5WdwpH4p;cxAl}O? z5Nv5FK4j6ks8Y~;0SNQy2Ct~BX~78~)_g*XgqS?DJIh=|TMw^N2VcsHgHX6 zy{R?9JY5_<6<6%m-QZPxhDcVUPA7a79q@xzE)A)8p}kzrOgB?qZzbrvfap)?F#DM% zTFDJ+gC~|htDj614*MgoyZjsu(CZw7bLcgQ@?0>XD-F8M(%;>TtLYOnpr)5B>8zfr zT?y!_7tXQz?57&SNIA%l7wQ&H(FCW92aCB$Hx! z&$4NV7ac5G+j0pS({E$;>T^rG;pxUvjEe>WvO}={c$6e&;0A8^-rQg)icrSuoo3y8 z=uvD7E`ViQ!R5qi8&qq`PCVanooN_*pfm3oO|WMkrQ5OJ(ofE;>;#p?dTr=3vC(Dm zWciOOY|R=me@Y3vcq-GNFF*w8WICp(3ADbLrjhBEo0kip_qKa=_fU#3TgsW1Z;PbO z4$*x(Cq5@Fnet-_kxOBkhbN6_2q0Oy#$Yfy5v}5mZqJJVPLlMW#)nBE-$7nPjLmbmX|1umD$da zN924e2WjS=hfyh&RBy>O|FCxo&j))7PjGe>G&dwI8pYSC>!osW#?!`vj08+*X0;U} zK}fQ4VTqMRjv`2RVEOLh=*-eTs@6GSwjKS$U*R_|0B6z-DyjLxRiR( z`s<`6?nALWJO$P1xJ}naTk5s##QwH_3s5yW%ldaa*F5BP$wAKTAg!V(en&*kBg&Gn1MAsL8G%+#fM0Eg}5lrz=i zUT&XAZ>_4FMU}nSkFx?ixU$pmRW9>U9Hsvmu3Ehcr`%xMcSkF!?%9V$ix7 z3^9Qywb-(GqNvsK^PpQrm@`dG$fq1uQnobp)*z!&>SNB5H^#28z!1HA0;5e8S68G{L*vCI1Jz*pl(EY|eu1XAPy+;13CVaqXE)323(xNR~?_ zuB5}7NIrHz;2rAjvdQY}rnm9L%3~7#{FWE{x1WjGmQ7U>msli~)!@#xDveDY5qw%g zke(p6T%}64-(!SzFQ5oal;AeHk31x`s6K1dV+7mkF%(g07c*dBjTZ24i)T$?5uC<$NAScw=N;D8O& zOuK(R?(A`}jR`+G5S+MU!PF0XAjf5_{>IOlxrm~0gua45L786oi-p!zFcDvq$g_gE z=(wN(p6rWzlObw5ylnLyG+?P?d)Pa61g$q`>}})KO}gIo(qOIr2Q zj58#N%NW#7@_1InYFf^7t6zlMOFy0voiELu1lz4Z?!#j@#Vo+^$tkW+igLxC5P}H~=>K zOooJa&+l)$y5i$&T96Ctsj<{V0SBkSa~v*i_^cY%EkleKL@7S+I7WnbR~e)GDobe- z#^|jvfpmoU*>m;qD01_jw62KGR-trd!h&hSI}dJoN;2~5+1(a`7?djR@ETJXaKCCZ zlXObVa=fc{#zN3^$M7rMlpqGYk~5SPw5L=%%yXsVjQ)md^fw@o_=lIfazS5i;XqQ%UZ+86ouss?u-(zVbFej1nDfu~n!`^9`S(7kWv z{BL4pGJ7l4T?njZpq;M5^}EW&5t|dRhtb=Yh3Tz}=?ROs`Iw33dTN@&#+yIpyxg_B zQ#($Uui3f$Y~oex#Cv{}Eo5|w3e!YPnV96-H*iaX?urH|NPK@b@iL7 z=)}`AQFl7(?{4Ip8Vlj=w_to3C0 zpVF9~G%VI>39AiBPQl%G&`r+XgEy#%=y83?*b5U^0K0?X5P?FP0@NcXGEEOl z)LZJzMS{|!y9(Xn*xpB(U3qR4h4hpknBoF{B&b~;m}dIv7(sGvQ=W4a_c?)sU9iO3 z!RFIw`m*bXWu*wTKM&QJqw51NqcscEMasL z8%Fj_qHQ?ypj0J&hYCoX9A~unkwtcP=v# zEY{}3h-2~EZ}cR+dXHC2bsZPUw0-*2+aF@p4{BwGUNKteK1KOaE8F>4z+iWJf!^ma z1Iz)0V}F+Drx>MVCIGf{iF4Y-E9T2XU9^BF64>T* z4{(oMWC9nSm48>Uxn#;F1wxpzGC?;PbfQ?OK0&HS()YI2Lj%>t`+8hE$N)NaP>WCp znfp|@B0g6bnC>r5R|_Ys4AYwN^(!XCc@xk6qH7)Ot2SoOqbE$5zPh#H)2JmgUZl-X zQ)I~egUd~q?YuLAy7`NmAXeM>?RV`*yGEwM;0>J^IV&Y@Bo8BMOjoXoco3hvpBqU@ z(bKDCp!|obJTK0>H@lPK8BGqrJ})~*F20S8q7Ki$I-?A=!>}Nj+@&ab+YN-X#qvxw zCEqIDt1kdAm?=81?=a7I_EWhopCx}&>6e{dWZ_bjo8RB=V25;>)vgF!WXKRBDYQP= zGh}J6UgY^|=WgX!LiP#E{4CkZglL@9Q-V^o?pEFa^|pw~j}{!}mG1Old0i;;*K0(`2n)tCx^;~5W9@j(N09?F1@3hQ5WjR@W6 z3Tf)US@OE!Y#r;(IwjNY=&v@*?ktM(^w7T4O(c_r3oh%Ip#vNnte6@F-VA3pqYJ_( zTLKH_i%w8aMZrdF&_TluNX|X-R+@j7u%iR2c8?WHuW?I#IDuu(w&r;X*n8-mX0S3F^A8buZ)xVIU8jbvO_Kb|(O zn(PqREv7Ft4-e0ey?$kBA?_I;Np}#PkXP$VSEeEA*~#OqqQ&)`YvrhG2*xvx5D4>w$pkZx0z4a~I##(=}CyM*oWsS4FXN);f{`eF~wENXa3EA65N10-y zohAR`)sz2=JZK)ofnJNyE65CU913TmDH^e=SIUS|F{3FR(Oj>Jn*71D%S@zmF-7+f zgb~3yJpmB2I=2?&INxs6-S3?h=`~d2d$oS1ijuak*m&vPLkd8u*#n91GC(+9PqL6M z#&xa(fgILPDGVgQ7M@h01b&$#cQvgzv~q-Q`Hkv))u7hRTQcQApo&esDkbX_4RZj3 zwQ7bald>msak}A*NN)rK*a~|*oIc>p0MgBncQ$&Tu@VU6s__)!0Gv2ckw8AX0Z@TG zNVx94;uo&ve1b-S)fJ&~>w3wojd!^^c_#vFqvuDR)ZGUiiPIe4JzS`hV?+{PE5 z4UZUp0?OZ@u>I!Fn);yizrHx+A7AWdz1VeA8B$vL^EkhLEI)NW24eYQCFkX4 zjn}!)k?VuskFQS!>pP@w89i_ONn{OP@q}V&@u{1nyK);to|v4A-HjU<(0Kg=61l!& zM~|zUb*TK*rguHV#-A9~lqs&+uDMIc!(>+GHzZ6fNP*L&?XM;ZaGL5id4GCzCtcX# z-5wCS{fGtH^zrO&wQo%wonm*#4)klha6cX68E`hn|6($L#h&)6Jw0{b(x0>GU3Y`=Cl2>~D(?A)us**0qgnpw zy8qkW@^0yuu-DIR+xQ?tl^@X%FZYWR?l$@@aPCjs+Gv_qoEBjUHK>&1G;qqa8@c%T zX+-nVc<|&8vB_qlk!l@iL(M90x+N3B1;YAVUd-!XqR|KXb6AA-x9%$Oz?p@ejX}=+ zsFgC`X;L0ASn-376TUR|D2&1W_7 z{Mp(EfxP*_`PRJq%_l!(zTf=n&zAl*9sb2&zvj4)Lwz*hhr{z{AN=SR08IH9SHEGZ zk8bgA^!op^TO1_3YCL-H zNrW&A|KZ#ND%>uNu$uDN9(Lmxg5kVcq|J>f6YC(ncMolJ0qGGoRN;2&20q}1k#v!z z^$!qZL9Q51&A0MoqD5RN?oUPRFKFnmFpWydbAMWLF3!9m98Hu6wE=A6uZtL}%A2X~ zGXF<3q*C%&+dg@fQVmD{yQ&=m%1<)%dSF9gMo`;M6DyNH7QlZGc(wr2c<#$p>d-f0ILUPG zb+BkOT$F4yQ_=EIkPHAfnEbbo%cDI*zHR)^;uT=uikV~(M&byw! z!EX8vZ67e}pLc1SLh@mtcD*tM3SPg8^S)ID;M#}!5qc)Z^U~kIsvg&!e+Lsk4)xK1 ze^EVrbc>H}@zE_p|6fkGSDCJ((eoW_py&2b)NH-?jfPO)dVZS<@oSCgJB372k`H-i zqlFLl=BNA(>yajw^ZyTc!YQO6{qKLvd9hAO$Eh|Du1nrrV)(@1^=)OCL)P-h#OEj1 zYusu-x3J;H>I&yxYf?tTnWN6I#xVFQJost@K#wmO2owgU3bJ9>OB`%hJEoKMTwvBl z?5)sG02Uu+h5iFGvHu>E1c=`b=Kwnctaly2F#ov`d{vUWtY|#BJJWQ*qPO`k(IS!} z3jsO}9*>5x4&d+Kffu;ZYo}`!Jk<1}-)^J;eT+^pac@ep$!wiR!BVP=1m9BuZwwr5 zm;>5{o0d9W&HzsOOsD`&VZVrmI~Q9K;XrM)z7`rPJbB%z|Kdj5?QH}G#g;X@I2IE?7t`EvhYOt@dw!) zXcuQwT%^H~LlhXE8r@!wq5BNtflgExC*6k0Csn@G{NSm)l;rA8<6;Q!R}tnYuZp7U zpQfX8&y5u@`0JFKzju<)oLg-ZVvi;0&6-@cl;CPSc9R=R@DcWECbuvg(yMN+K3%9* zqaYggFZY6KV6eIj1rN8+hR?R(xmUC19x;Ro%DfgOizbL3tk1q2HrkkU`)ELh0v(}5 zZ-zHm9RDy#FpnTQCKIYW%avLV^EZku!RQ5R>KGIm-PDaBIP7T1^rAP%;NM<^&3zXv z=(mT|0_vcDgmTkOIe4xcJxnGM>3xu8mZZx(=F}PC&2@_Vwe8CSlH(@cWQH?_(qoql z{xoVIA}2`<_9ZJCDSFxqpvcKun9gUT)-8Bm1$>csiR52HLO3}O z#R`44Qcdn}*twWjbc!f&mkQ{=3co3FzS6%=Xe4>uJbk?`2?fh6@X_hG$;T-J!d5 zGEUHa2ZjAy8siQZm%T8usU19O&V}m-(D9=vPAR+42JOc~0V-i^gd2A5VfM(Xk=_`l z!()Y2HqgniWRbT!1T`*~zeNEtooCGs~rrUthPj}b;?%TZw9I~LDqmYMNBLX%e6TVUC{;+OA z7)tH9s6rX^?B#yEsnzcl_6z+FILf+G9JNX~#@+yW@M*%0hJgtDq=uXaoY0xb}X zCPUfutu59*&1>p_Mw)LFE-!1o2zyl_;D{HDZ_c{ufpS88Zh|iDirNXvJOxOsy@M3s omeha4{ir{H6*e0wy4g+5i9m literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_5.png b/end2end/liger_examples/medusa/docs/images/Throughput_Stage1_num_head_5.png new file mode 100644 index 0000000000000000000000000000000000000000..77058dc0bb9a17f51914054e528f072f7160eae5 GIT binary patch literal 14364 zcmeHuX;_oj)^?N%T8n6{QlLQ9R$CFMGAUtjz^NFpDr002#1KQ0A`pZC(PFiVKrLk^ zpa?{PFor}30g(eJ5duU4Nk~u`0|^lViDV$}6WgBSIY;|m-}RmEeSduYL#~VG+0Wj4 z?KRx@T5DfF?si!BL&Fb2Adv3SBQ7UEpyl2m(6We?+Q5HajzyC|AiI&HF8fYh^cV5; zs!m76sArG$xvVs}Z%aH0FWdk5CWx5}an_R69h$u?ovf|iW*$@*e*W7166MZ&>y98c ztV15W@*(nI@h1nD1~Wd~@V8%FT;Z3B4zApE=$j8LMzw#zwyKumQY5_L`T|u%v>l^f z#>Vn1vl(Y)c@YerB2Ses5wdG@4ORi`sW>yhmSup&L&>sA4?U@1~g!6_*7&rZ50Ls=s+);>+M$>@)ilKumu;b3;E=*hN% zFniB*t;i&8ddsOL-2&z4Hiew$EbmS2%ZBA=C(NZ4SeGnND{AhS|ClL`7h~IoU)5E{ zLtsD7zG~IePfkI0lsZZWgW)1h9c09AqEXWA>h3V>5*(8{(R+5VCdAd37Q3+5fmM3Y zgp#HjB%_D<^PgGjOlBYw`#W9SJFk!H{cvAtBwysYJF6nY*EY^|nsq?LFu%@V#wj~+JEp>0R_GTg#LdnkE~WBfx*DgJ;gZjF znz-7oWL(D1)DtdV-nlY0YL`nz>se>AMw(wI6yU$qUh{!&M&fVNQg|5Xtd*YQe9r@c zs@yfEJa{{s3xyTLW8^*8oPNcK`NY}-?CRjd-yyj99=^5bAhTGLn9+534~J`E{KJ5# z+DVg)%au6QbemK)G~uMbRPC1WI4VonA;pX12(RPSN`|_;_EP>8c$IX9ms|1R<&Bpf zV;xS?>PXeXKq@vQKeLu%CgkX0v7@d5@)N;Jj~^dtcyZOJuYMvwcj=|&QsG7JOX#rN z)6m!$xg(h4nD5CH?PS{+N~J&fMFhd(!r~jkDsin#Dh1P7#dO{8{{k`|d1zyyEO$pRkXqX>0&C(3xibE-GSg<51 zKp9jma-O@DT7*uJGP80-$F}5>;#Y#+Kt; z$={@Qo<`V;K{0$mhS{R7xGp_YlIbg}p&@frcqwJ@Vok>xCdtBsJ}{>0z$veFFOU7W zq@~9AL=#?838)l=9{*x>z*kFkUbh!FT-ANutl~B!AERW29Y*Nb z1Vb|MKMXARgc)&_JKWJx#>eZMDEwFiQ~4RzgOiNLDY$KOn5vv3l;+CJ=KeJ&SVMvf zm`kzB*$yLm2nIx2#5jKEQEpH@e1;oah-jJ*hwcJ@HQo4`BeaM9db+r2oR1YFhgea@ zKQ&fd&nJiy2yP4+(YC=!v36gXTuEsC#vIaQ`SVq2M!|oo>EN!|I{wz9X&}^eoPfU&lY#>fJR9{QF zs#a%Y(jHM2Gt?w6Of)abD}1vOf7H_|IGvO|THGg%?iO0jxL{BkdWE|uF2meq=J zY6T>dUi5ld@N}F?;ei)x{HF{|r-A+5-wNO{<%q1hu z3@xb-o^^xR^IUW@tvbw~fW$)Eoq96@LO5lUPyNHW*5KEJa-nRj165f=Jp;L7j!vF* z0hExYBrb|pfe(bjh+)_A^>)P`-?bRKj$7eTE%I}E%^p4nOA`uRP+9c33)>4;)f)N` z!)O@W_7Zl){qoG&0oVBSis`+hQIL5DqzoJt+oaCKEoQbwv~*kejG&pqkJ3cYu!jC! zc4x?8&U50NP*avG5zFS2#kB&~3* zWu2QFAtho3ygtoH3%q;U?Q6STW9~Z*v8Qc$-B}?q z_%SOzjBQ5=o4dxRDYTxstZqH_sWc;xSnDxz`*$%NyJKUSt;&Pcmbi{ZO+uUa!jBSd zezCPEKNOQYxa|1w$6bnH~< zqX3^M?DsiG$X=M3i^qE0n=8w^bLUPtQxt2Si_+T4Tp;J7DvNxHOzj*C?(}fraP6rK zQ-WXREdhg^TOr%_i`s%VPA5UVoEN&#W4<#FMpF9RRmYqZ{T5?h4`O`lo5)N$(+XVn z3xT~%-lUi@^M65Uur1mE4rDVp&}`nMRJJfj84sb>^Pdc2NMoWf&e`g5rqhow7)1CS z(cIjaEE(=k>3^IPrHBE8`_3==w+C1*cr;XK-5Rjzm7Ee&a`VNR!x`JfUY)@Y`j<-d zi6+>xDCx;*6qzDYgyo>bL${~qPHw{89;sZKs*K#&b0SOIRWKhHP)Yb^@l=J$;9Obp zb$>@P=hiV&w`VE@c+||ZSH$GS;x;de`&N4a2Msxu0*lmD*Z6Op&a-t`lQaFvyOsL}wFT;OyI8i3ZPznQc?U^wy;?eXV{c$Wg{~2 zq6(y#BEL#WdSzU3?`V`{qi=#@{<4lqkB>2AUJ<(pmDKPO4qibXhBn1*tWG~|*5$&% z)1J<4oAsq_sF;%_v7va3`7J+2a6nd0rlsA-qWlQr9|zZq@&iJOI66#bPcsrFcoJkM zrv1s$CUy-pQ4AsV4!&{7Mc}pMGU5`f`?%y_V!AWr?!d~i)hYCW6{B9?j%j%(Anr9L zU~?bOOfT3h20cZYLy0b34E6l$sPFnbP&qab}x;gh$^QPxfB!v&2 z|He7#2gHT%6j6s&sNkIH_+qE4rR)fHLV#C?T-TMC;Jtxk+HHI_df{uz-0eDd z$)Gj$Qg4M_mHSiWLV!AoDM#uA$wcYte&`@Nr$JJ;oabPR&lhv;MIWUmoM=bEcuw7M zo%}VCyvMq!`0(%E?0aFB9ZV57Rl~>J8`zA|D9$jla*b%ek3u~Y2AtuqQFu|wgxq@s z);z$1o7R!xQbB>6&~P)f+i78th{6R};t)$YE2*As6^m6?r&t#esv6}3(Uwt%?PWD4fA+XKq4n`L_5cmh9m=aq=u4DLPLD5y`@g^RGC~P z{C4r?mH6c@nNM`8=H%RXJ$JXoli$c|f!!#GtTG)b$`pq3-@!XQ8K!d(%370A%4-0Y z;N+ARfahMeE7((80GIK6$!T?3F|6}=VhK0p^gy~*uiH+$d|I4Y?C+q!8?%mR0n;h- zL>rAKUbflfQGIBysrw#joJB!$sLWwI?yK7#!usrmH~Z_P*ID%{YcTYwc~l3PFOH72 zW}`O&7E14o~?FYlTpw*=FMCYi2%0LzUCO_R*mjQ;1%e~k44zi zzJAv5O0w`$$iNnMig`ds;WWkTJLwaB?1m5mRYdilldWMh&Jh#H{qu3d{FKgsKv*iX z+jo+Cg6HH0uHr1)gL9p85y}QH5S>;r;eI@pmuWzT(Wl(&=P;@i{9;w@=VUjkc30D{ zB(C}qp+8cg)>am5_J{n_JnML4__FB!8Vdr01A%!N=o4v=*G{Gu_o+`HJ0Xe6Qq864hG?ZZV;3dN~N~n`A|j} z^CyODJQ;Utu8(KdK|Kl4VDdsjGn3jo1A{`uWEKsD=4N_J;0yyUKVWGt(`eweAoyO6 zpCTl<$Aqd%mc}(Lj_2m)Pw(B9Zs*N}G*xlk1hVtnF`p1erwxU_I%*GSI9CvulTB*a z{qyXA9^&H%_YC$rhgNYO>xXinhcKOk{!yH|j~8D;6<5do4+4QN?3&6HBmHn)BE8#i zuubGqK+E~i<7RWhpwH@fg$bx(Ut<5EGPT;Q!!21R!JPnx+%zs1etu3J^y-*Q$o`PnSuCG#2I137KM{O_8>1c6l|?1%JA2@+!Dxz zG|vRv-w#suU$^}Zzx=l0f1dXyo*BJmEq%sX6--}-cUT(rme2P5t$GY}JHXm|C0h8x zj5(QHi_0mP-dvkBR%UH)Uz&E>gSKQ(U7YEi2Ty!N7{IK9c4t(6%r(qv3awhpXUgKs z4{qsqA}d`oEc|jz5?;S(S0O5CwvMuT24ui5+Alb#3)h?Ee`<&v!w&B-9(drFKU3Ve zRO8lM#&pmlvT#qn%q&X9DM#$t8XEE0&JwlivWfy&2TP?UB9XO*kYmo*Z?oz2A0%LdsLD2ysd+lQkV zpK^Y6S+A(N`U%zuVtT+C7_a%R=z>Sju;V?3R^YClz)g+Y4a<`(8ot8gmKNJYgn|Tn zujC2_~_4Sc8R^4h*McPF35FVE`S5Z<)v{%Xzs*4Eax%cJv5cT-tJNJF=40hhbj7)8N*5w(qdbU z_{WvB5L)@bOhL?eHXI-@5S!dkN1f6e+K65Gt|9oThZ4|FiQ21Qc=QGWNdilna?H{a zRS?aO5omZy5?3nTGm&?4D51a1-(v@Cey?{EZ|3PUfQR?BJAiZ3D_eHP{@fEq?n56%R22{3gWPw*lwT$rq0!5(#m%-GPv~-u#tr9uG+1$#^Q^IM~)ry zv7AOmIlsLmQHfE$&fd=eTd2IDP0#!O(JDBwzc%PbX=ODoKsDv1uX~HOpE198aEi>w zElCQ7FD8YmmsBJEccSHc@R@bFs7{GcHk!x;(ge)l8pR$4mM<}s2otCXco<&+2>+$@ znzQmndvd&kUFvKn>xU-{5^D8Y*>L`WAYrCLPHMC zQ<@ft0(bSiJM*y9+^u%foEwdYq@iKuLeak9f!4vgZ+v3*fY(O*!dMz{V4t}9_wA0D zeV(Y7k_&^`aN=zqx=d3NN!1i2e@RM7Rg9jtoEAkkXD-a&E!8gjZT_nh%hu}XZhTZ$ z+9DsfYxWVAH_xS4-poyC+u-#HQ#2y=i6Nk1+HX=%kO2keAs~`w-_yZCXu=?YxC`2` zFc1)Mx;2Pi2gPMCRn*KmfNQ@8@<5*q@CO_8z{09_2PzJ53LDT~Y^e9P{EI=s{*-Au zKp=WdD1R=ay@i~)fN2<}$I8F+Nc30$Xdf@K^$#uS4-^en-`)K%y%M_<1F&0VdE-f( z7(1Msvis#vB>*h$KE|%wzIx-1ue!rVDJjRA?#U+arn+SYB*btRGTFO`zf6QIGJ{<+ zc6@CJR(#=k4(+9SVY+n2e(v!W=5&BsP6F(z-9ZKI^zEWfm6qMmx+4go;b&wu`3a}` z13gryw=+cyU^Uk@I@=!ziI#3mig)h?GIiy(=gUg9&35JMJbZS$W&g$CRDPZmCwoj% zv$kyQcF(LGn--FoZf4y#**~_RW0|>qD%o&l(leGZR7RlSQ0}o@*tQgN7+r6><`aeg z57V3VcHUr&@QBs9r=Mnq0bC#c;?a-r?z}y4L0>UkC0|JI)&haDKAh2I*$kXc82*oU9da4&3+A7~oNB{y zOUgy~*|S9tkg!~#@MOB=SHDZm;UkCkuiSLy=MwLN=f!>6^zI-Pkhu#QS^DKm*FLF; zo5TgO8m>tj%}f9O$rSskdoPLm5EUre0xNRrKEZaJ=v%j)=ts5H)zc%`AXr7|W?r85 zdLU5X_6K&}tkbG-^wPQ5cB}kbDb7o;=K;;L9Y+pUZ8G<+`n)UOk@|YRApV+1`X5pQ zGS|nW|MLOmln0u)OuCMbRlvKcYv9s zh>7l(QWg=)eA$*{hS$0691FU+_H|Fgp_Shld-Mge*=4!truzFWId*7&AZ=`p67#5b zU3f61FXmpiNH*iX4R_9+nQ~!!P-?4*o-#BUkT0oX-ST6HXLLQCupy)tAOw4kNCyYK zbM{1>E0#Pt)-psc=21K9mfJ15&`k`a^BLT9(~@v zLo1b`$iGX*+IuqTUuM~2xMTj{zierp?nZeb`0>iirbOghY8u~beD=!2RF>`j=Le<@ zH$9AWN&K;Ca?P&&&krMv{y65smi+<$y6xV;JC^^vyo5h&F6X?%hU$(psf3Eqsvdpg zBX5cJA>)aDduYW4;q0>wngu0(_0~k1ogVSieAxcd=6^`-&5D0}UF)yc|NoE-U{txb z$ZzQx=B_%%k;iAn<6XHOUC;;LpUJU;x%oCk!hM?);)^BPyumGYMZM7cf-v*QfZB?{@ic@U6rzDJS$RAoLSOgigWM-z}dmH^mqC=bWbv0{=@ho*meoC zBTWFh@su0ACyX^7^(uOXVZWB6cl| zfQ}n6N*4SxF5us0j&I$+%XshGHm9-fsnTDx&8RPLnDtdEJ1ebc7zNJG$v1C)LJ@_f z;p0m$&|=lQZdNzc$S7} zb*7vBnK*z0MK!CW^N;kKzA?V=c+(rw82hS4$4K(KgFSvaHzYah4f4>&+uE)B<=rQ_ zQK^aQ*=ugU(W8?xRv&f5Dj{I6lZ2d3YQ;xCc>o1gegB>yRq0Hh2_;bpw> z;ee47G>ob%*UoMn`A$fi=%(>vk)VTbK_v~oyKg4bp1yZC<=cVN5s=uqC)=2sZU+Cw zPA`DvybJH5d){Z}WBkB$5xaJq#`z>GFeUJbZhV^v0L&$AN%~&)?;q#^Ppm_7$@4Gu zn;(SZ03ryFj3hmvMT_~?(KAEePUzYLJmd@JvQOTEnHs=XRIwdd)eYo@B1LrrwljT6i#N`=3NFz+DRwh8lkOa(Q&!`TkN8M#QmpaL8ko`=ohRoc0kk z?jv+TuDmO5rUOj-^^0o+1FLg4UCC!!uaW{=ez3+$g)f;nsj=suEf&BLukOC*vm10h z1K+pF-oOiR3tK9n*kZ8!fdi84GRDT(BZjAnC!_UwB zu%;&>Uy>II)LckmRl;M;Zag@N3$!LclJqSrBHxc!OY2wL?*%&Xj?bdYv=RR&?mCD1 z`8J|qxlipm8&oRLKACL=^Zb*z&_A?d54=CH>;D>=TSxD6^R@8?vb{&S6tpE!L?oM= z`;zYU#k@BxmW^9HJ_>B_omq_kc2w(CU3kk6E8tQgyN#e0g3C+u-p^xzqgZ?OFIfG1 z66P4)Lkc=p1l##g3g1-j@|Ie65WD zAY?K(I5-$S{xrL|J`wUB=wEgjT{X@qC^&TJ&ZPAL=8#TEZ{erJZ;x?dBKkHjy z6wtQsa^PJJ`$c|KOmXWP@Ud;t%2X1XDk4Wd-Puu;sOPlX97^7SAR(t07Dz<>JX`f@+yEe7>w6n71f6 z)sYO>LE5Vha}FCvooX1*LELBo`qn2(yqf0Hiwl(5BPrmDvDlq4_DF&o*A7vq8!&sr ztLRPh!@{$Z0-&N88{38Ow?a8>^(sR?gL9jJg1noiIL@hsxi@IiPzJEP-mK^sKkAn- z-JNM!(Lf$(BV=RAAwoC{-pqq5FdWl|0TR|eqQk@<#!=0k0{Uf+2v>yEV*7`rv&+tN zH<5fTORk~_*U4xq@?y_Eoek5wq^bz%`A<*YJq`j#O7}nJ}Bw0jc%smrD1SEd-sP z#N>o3$lR-yvHfLzQ3HNn%BMiRmr?fo>t`sVC=U6WNo**Gf({5*+&XMx5dCvJt2RGf zL)C%|uoXkm!|};6S*P*sJ_HqpAcB@Cv|c0l_V$R8ypQ zFtSzCVElmGQNq<%k{d9q`=?3C_DJq}x80lKMTh`b=cUS6^Dj9G$&kA7J#zp!|L>T(c*PXD9g%Ua(uOlq69KcTtf8NCWWdAbQ~WNUQEh zYVMnZ|HlN|JVv9e{UR^^)iFxJJZJ4O6Q0wUMj0}A5e!E` zJtH19(gYNKjlfl!%LsA7CR@GiM#2i?%TNJK6mT+l%9`_c`}d0GP2t>*_*8&9wzvH% z0#1FaJ3;geuE-GBDn-LfqDZu#gcBY zC2hJ;D>{>N_d{P*{SKDvSx)9h$V3zhCb^$x;Fkk&Hq z6Y_I3S4@(Oe>Sih{BrrL7XAkh_g)mZ2IK&DdE7sN^tb9c=Pix8FMUCi(Zv02YgPhx zkSv7|&_`Di*Vu;+8I;kmvW>8s8cnKQ#TTpm?dN+AFmw6yCG&qZY0NSEm zo8@KAIhrdBns8dU?_R9m`1}o?6ik&>TByFbWPiTdZrP>1<@aNT75q%i zj27RTVBpD)W>Tb}FB0fwu^DGCcx@|)yD+=zVI5+CjJ(u| zUX~GTPYf6!$d*0R!!85W9nkvV_IY6l=-C>es$N>13B literal 0 HcmV?d00001 diff --git a/end2end/liger_examples/medusa/docs/images/Throughput_Stage2_num_head_3.png b/end2end/liger_examples/medusa/docs/images/Throughput_Stage2_num_head_3.png new file mode 100644 index 0000000000000000000000000000000000000000..2595387fd68202192af50ee97afbc464e5721bc9 GIT binary patch literal 14905 zcmeHudsvcLyEjcUR#Rp(PFb2-W22qS4yGjqO*NTgYIacbgk_!uG8Iz;Dm$B6Sz01e znp$~2L5fILYDsE}iikjI9sx;3ND%FWJc(04e(EQZ$v2sVg<3;{o|oio+7SR;h~5~ zRfxvd8_v&ZTi&oc>#)&`odGFcx&G+^iiXys8V8VTM&PC2X|?d=bn z+ZPw5kZa*~-Z!QK2q~>zEf81RD`zcqckKTD?Da2aztgCKlX|vsp7#$Ab7?ZF4A$Qz z3+me>Oe>Lf(VEx_9BF$rW=SHL(Dk;(9^mK9{rlBlKp-!#Y0QN{mTl5q3;ufTUkL7O z%=ow6SQ`{G&XZM9a&Fi3MIw}f=>o@|V$%7}%I8mdMN2SZ76_nyi9;2Tu zl6z=dqp9??{GAs5q2A7l8_rabX2bIKuneRcB^zE=cqL=Hd5p{B2Y1P+2 zO^aWP$<7}v=HSM25ZePLyvUg+gS{WH*Eq}dlw)w!LFWsjy=M5WSV@MVsnpL*q2Q~q zjtL3)sh+_b)&@3B+s5`;oqWG{lCM(m_iXZ!)%=8;*MTL=G0!zE9~i1W)(x`#NT$l9 z=;88g7ZhIL|6L?WQ-&=uDpM{ly2y#=DY<-whxd{%39nZ1l;9eRIJodNT3jrYrTHYa%9 zsQ23fck|sL?4C-tsnQjxCH1Vs6BTTj$eh9>oRg*75iXX4F?Y0(GUzY zXVgS4kV2GiF2Qc0=n83~=Uur-DEb;Id(JbMnCge~UGTz=qOdyshzD&s&N`P)ijc8T zapk7*@$t}hn2RLB z+-*3SvE(w>5ITk2yp|~%AYVLy2yzw9zO;EAg^O|XE^n)Dd{;E1fgHdg^P2++QcRy8 zT+Tkm=WwM$iQBqXs!$^`0I^Zi#3ElLC zRUCexbyp=8TPBkx`|*%#w8^Cm(H z3}MQa&cXI=BCS%E8O>9DOi;PxmY7T-RO1MlJTg|l1=;?7r3VV`txZ5RX)R1f3s_Kv z9;&=I2}hB?Fo89XD_W7}R9Yt)=@tSFimwlY_98VitH_P~W5s9NbvJw4B@?$i=9F~6 z$m5O4mQbyS?tD!>3yGU}qR*1fe&PYejFxbD#yMxhCgl|qZpfR(efDTS?N)7vZOu=J zC3D`iqxn@H&OY7^MT+?x)=c79ri{}ae4stt0KblkO2czoDGTA>`e}}OO^BBcgglnf z+fLF^Wz8b?rP(QM{Up@{$<|Gns+!v2?P6IrjKSka6G_4dsC+>@!r-zpI46wZ>CzD! zDL)duW{Db)3>fh^ImI23fKiupItZ$w17B}V7nOf#30jCs;rW8FNm#*$e zV(0k#6+A)SwS$qj%dt4V=rG14_evqh&B*GyJsx3+&)MutejUG{(=q4yB`r(iCB$2= z@hTZ!M-WVxUKNQ)-}~VS`gX+`+PwA26QWy&b4qlN_+2-ZcZ?FF{NaV=S1Yf z$)a$T`5jtIs9K#e=aapC)m_nIngYcU@lVLO5td0y=%illmE#emuKBE4$IzOcA_CEY z8R!=Cs6hY9NbOT&KdkLjy}P5?i{rRvZYb;5spzqHmkd#BOER{SU-K|0ECxG@?TuS~ zg)olWNpgd4T+EO}7CuIqysCF`@reGnE={wI>bH_4I7Ls)n<* zh4#l>oO{RH_6O%HqBI#Snb#=u)jH|{$D-*uCw(h9z+UW2;;f!2(KTCA@ z{;2cTsP4za0^mCj7cfO+S5(1+6eJUNIGrMrq6@_?tH$I3{Xtt4Y$wL}zm&B_SjjbaAm%nrTg-rY`%q*y^+i3 z-5U~JB8X<{=?HNy%&)?}-xre^R*pbRhpIh~bXXBPrrKI|6c`XRSIk6~u(1a#x(XMn6M z?IwoEbD+T%pfytT=LDDKIhFZ4BcPXFHD-PgMBOYW*)m_K&;nE~OFmLrJ)0s?Je{*=5X z_fb*k72#(Dr)L#ZupPGy)dY+%Q%^nAC+vcBUbm1ox{N|W` zg9^zmsYP@6Gd%-nce*PwhcG#$D3K4VM7t*vQPNFSc5L3|-(t?!!@uVV6VKu3q8xZs zwUmP7r!*B{ML{(b66eK0!i=EFU~Kt}pz$rE{qfjy?c}5VxktSF^jUM zAe~L5_F}&Y)$5`6yZs2?iO3d59{&P{DAhH(73B&Gv9PnLbP{E%cO3bBL<5-}T(0@P z!qlNeTg0dw5%#$?Lls!=RG!6AHpeLnO2|#2AqNoDqjq}>T#W9?vX@40OHSBR8nyH` z{JS@6r5$11ZK{e#V*XSEUqX$`%4HWjxpIMQlU?nCm`_o_;jIgn>bpzHRD{7g>9!y# z{u@_G(frbbu^Nny7spLjakQT!?sv+b3?x>kd=*{hqn$)dI6jj=s!QikoE@dFbfOpb z@>@iF=yA9pMG;)mi{tW=3i(0(c^!(u@m9C!tLg)WAwVF?T#XHTL5-k zxfSgdydWH7AEQ4}n6Zp(2c6YWb*q%AdX~*TDOb%z!r5Nv@iIn3)tK$Xjm3(O6FaOF zSfVnoR^t@GMY!@Tt<4|Jyls-dk%Pb-+Tn1%vBBP#A`g=DhZBjh-uN8-<~MbyoVpv^ zNkUs&(FWvraTZs%nJTWLKBUp?4z&-CHHEn%;I3Wkd>WF4c>BjEMn?dPo)V>i{{&Z=SYly0u9rn|Hi&aFSlYun<~1FqwG0$ z&%d@QsBo9`VZIozlGor$jRG=i_NU=vj@mujVYm!(#@D~l^@Fy#$kYhx{M<9Vv;12n ztP;B`l#*`lf+>Bj)PhD|xI9#xp@BIWU69}zciwZGO5?dQ=sT?30AlrHOxw;9m5Skg zsdm=bV8%+#q@q{##v9~1Z3krvR%}1ttHbh|B~)f=m3M>+8PL?tT_mjhR*tq3R%*d@ zYI&C^;?O0+?1Xepf2ob>cC9bkyySND;SBqzktsd9(kp>F5ZaG1Ux7+{iRK~*WQVUg zU;`j`o4sa0AjkLo=WjfI*Wo?*f(g~Q#+~oqf5ph8B|&TcQmQ^=-XhIK&!8g%&IkD( zJ@^!oU|CNrM|J>#Ag!x^%;Fd}3{Q9>ChI~C#Gs<@yE&|r$*|S9G;YH?p=GW)W11B z3>2ZYR>T#qNO~@NqqAqk+oxwGQ2H2j&>27O%;WY=Ly&tKNBv=3i)m=DE-7!cB}9;ISbS5qMbyGs})}m643VMNUd7bPxA+T0>D2 z>MBxQ#*ZXjxBG7RaZdhfW59;YU#R`+wzFqsOVjN^3qn9y*DV&PJ;ZB_HxeEQSNcWG_E6>qTJbzdJKCW3QIIM{b{5qllBjqLV55=kr{snugu9#0*p^h^*><*W#)4S@dV-%~#$Ru0df;5yy|AV?sDNgn5b>4B z5Vq2zEbws1#O;F9J|WNy?V}SXbBqp}MY9>Y0-=8rDaJeO<{P91Gq8MBK}qO2gJ_{2 zDXXhIwBJU66R~i4Aup=q^DC!%&Cpg!a1qNaDvNN@ZrqimY6M$bvfjsTOm3 zFa9jmFOrY!QNE~h^GyTW2at19J@Q8gXt%wg{_$i3Ur) z`&vo@hdO!>Wvzh&Wvs+4pM`v;7)_?Tr+OblXLn@hr6PNz9JE>E(IQS+C+@B?zj)LN zTKr)o0bjXAR!iO`1>9i2 z^-*nJT?W`hMDCDvE)mtcdoHTOh~M)n*o0IMm2H zRwQ0kAuMKuwLNUh(O9$5VAj)@Y9ln6-<6oAXx}=>)-*HN0;dtVICE;|l9xsa19p4z#KWg7XcGr7F$u7(CxjS`{Bid*1pYtLscWNS!ueQP=2#OPPK+9Q z6foRX4k#yNTzIrpXagMB)>SCL^aRviTN3c>{G5Qs)U5$MwQ;*VXF(wAzb&?OVP-ox zfCp@?1*vCjRwVkW8Zt18M5a`Br zTJ>t^>5nI@M5Rn!mUFJljr{VylonHEg8qe03K|g-Sr@M#(4Kqb*Ri5Hy+oa5YdlW9 zyAUTB>u)mSS)^`@JXtZ29@_$(pq{|kF6Gwbl7C9 ze59w2Ihal2J%Rbu-wFd9V0W?pm13iVyF9XmE94CXOYNPrm!Fr!j?XEVJg6?FXqeHb zM$_K~)WjbnhSV;J6O{Nh*{cbab)2TA)UBZ}1t5s;*(udvRAn9IRYum#8Lo#N^+#*E zhL#3r+PyCe^clgr$1Vg6!HAbK%G4IqP>`PiVK;3Ay*Pf8qL-RuezX9&PU2 zGaGVf&M!~X!PBms*)z_pzF+<9-oSgER{{Uw29dQbL+aGE4CFJpWjkz)D ziHP&d4D*oZAvX=zfKqs|4yD&GpB7f*lCQv`$5SD5W6_wV8Lt9D#rP@6T!edMBk&>uCc|lTI-B}jo zT=C)ATs@Y_AFNwASn>FenrQjd)Pa`@a)*IUAYCXX*n#zEd$jZcFjZ$`wOo8lutK{9 z)*5P{3}9EBzur{@Tb`V@*~N_6 zSc9P3+X@+9X^%v%=b(e_7J zyqdQ4fY*06nA9LUOIHE=HjOicSACR_u$K;-u?`b|xAyNxo<48NV!%S4Ah6coW8lCr z!*9{5cfx*I&9fV)fpopjhgIjFzYT*wW0hY0^)&GI?wPi_(Ki))X79H)cbUfMte4Z) z_oIO;Yxazx%NoDJ8M0*B3SaIL=q}%(-|Y3(FUZ$lW;AV$k^UY=Up_R_{S}Ebk560W zGOfIz^N{e}F$|zUN<`w$gLkO7p+@qV*fiUrk}gKP0%+ zzIo>-li%_83V?A7Ft^_j;6k@qzatea^?}V;J3DpKRrup|_WvR^at{C8;6CU=GMb;Z?#bJ8i76%&6t^O5Ff)0m``Mqk&`Xv1Lu!|v z4J_wS9Egso;JQ8y`yu{Un$36p((bnBdvHpt)mx(#eJMdTCeV2LMPPH;Rb` zof=`H>!N>d0&l+}tBw zfX+`4+FCVN_Z-lTWRMw4(a0&eN;h z^W4if>RW}aG(}WxGxe-|k_}RP5LQC0cI?^P1}e%I0idd!&SNu_9{Y`fR(@ArD@(+M zL%pC50pd-kF`!Bf0oSj&@~({%a-8t=QafXM>U5o%PEb?rt(cc0CDs`4`rI6j|7w=a#&VCOP2Y?o#US=vv}y zXLag>mn%8=i0hLR@BS(?e%|So=Th{&M`*+5iYH#y_oX6^vpcWI+ZvE2?f>G%1`-^u zW@_xxwr%wYYyK&qzmXUho@i|MGZu@j>CLcC$O6peUuCi8xti@q{D`-QLGGEpyh5dv zGtYfBS65IQr^kA7y8FcybBwU#uj+)0W&fQC`8Pp@!j#Se;5gDsf>z4f;T-`w97FfrO2Fw$MK%k%RA zPBSRRr=cyfpO|dFwfB}enpYY2Inwm=UfthCZ*7?M->&<273ts6?4OGCzg6;YP)SC; zrHdN{Ww*)q^1jw-iz}`}3S=;Bz zc*kD7+*Jp&9sqJ@VX|m;l74ewuew#iEb^5t*nnEFbFyagygQg}L)vzzh)1J7sSIM} zn}_!8D+etkPd_|99a9!I*iw43w?1s(&Hd8T6QiZ4Mvl?ufSZNtn_%O}9FWZBxT%lb z+83lWb+d`lqtRblPt554voDPcg-F@|ef};7)gYNYo$qAuyl}Jm2WpH1* zBT6n64W663ZoqN}0i(Jh1?&y8B3y9QL_E}%!>rLkxKF+!mA}7fn}X7Lp3TJtQt%!n z*i+3v>12)zSlQVmoE7L>L)g-l0y?fGYF>e_{2{jrs}LYasLGMru#X?!g^e{>X6NKr zJT4lZ6cjjs4hf&4;T(h_u=)7W^Vp>|NTC9uk$- zozmT8=OaGS$DJDIeK|ER-zd!AD3N7aq_Ncg?>HN!ico3bVxx>USRe&Wl2!I^mXhpv z{b|uCZ=&50!4i#fuX`CF+ZJSrqHyRwRtC%H%IH}c^*ma1ES|945voWi73oRLDnC3+ z1g+YE(b4zWm<=1l;F_g)H<9{8$?ZWh2ek&*2R7*Y4}X{)SeDQ!;=D>U$@WW&%z84_ z;hjJSn%Y;l*Bu0;Rh}5_k)F z27A5_6n_|*g48kEB|At}a8QP@4^L;-ly4_@RlTLK%iRWO7_}maa6E3Z-nr@~ZmP#H zn?)NAH_0nu;73jHH^ISgRC1{Y`%DFp8DfC@`6e&1{ywCzI-vP0$;Bd z!!&x}{-u^Nl4`0p9JE34I$o$x%Z#bDMBLV-q3af|N^t@J2m$~IG>=Xv%fYhrRE_mD z1f}j{|6C$E_OX=X0Q#u!4nOCAjN|W>@uJZqHY=hpAPs-0d)b0*Me)->H^N}M8`>@b zt~^yC?F?e-@jhIvwUWQMkF9uZjF5ln8mWKL`5(Gc8^AAEI8b5|NCnm?ENMbb43+1$ zot&3|kv~#v6Fb0f@;0&O&`&2fzGSKg!S(3z_nP5GVq=X~NXIp$*qB)1+kub`G$K@C}{o?@SvLC|T1_U5|rC)>lAc9xkjv$V1(mA#(~K$COD_Lt`C zuh!iXR1CR>%Cpvn%zh8fLD`B=)j?an(s93D8?R4j+%D^q5XGmni2LgOq6^wG)n*J!1I-md*7zXyZY0@<|h%gefoXM0S}f1 zg4X8zWOlhDCb|Sfr6kS$pgV~x9q*S|ooT-02-)#N2PeiEfenv(f5ZB!k&D`^j+AK* zYboDqwM5Y6-0t@>2mB;+5FAH~^yaG1E%;!LRbWR856^53EDxYy6LkLljAqgP@34w$pcSk)_AJ1Lue(QYquJ5k?#Ugv}cfVD; z>Zz)y>Py@EZ9>aLy0lUmhL7pDg*b9Q|LZ0LwXkMhrY@J4+Q=-Lbar@%ko7B+na0_{vu zSqTE=@A&XMw5YXYFN2 zc3Fl^m&tD>8#cPa23#vIIaDdOcjQ?`^`iPbv#giiY2fCl5J-bUCU64FChcgHrB2GD zHZXeWso%*1X)2tzO>rD;_QFxg97HjP9FCYMO5RQ0=@}!U2lSPDk|$TcT#T^nVb1sa zrl*>pqc0aGsgz!8U(%M37|MO@DbhBGJed6U#>!qNw1mbWR+N_p?F&5J!6&hyGBq#x zY!)grRaG}U|1#1CqCRM&VRi2u-rZzmF(-smlvQk}8}Xb>h^?ItrI4P8BHVkKv6&6u;!nS@IT3JHNL zZgKPRKH&fK+xxF57xXF8YGzPr>?P?uKi&b{Nj{s1vM&yg;=NdDXfKEnM9~VmaO-nag39#j+twFCmlq`G%%yjK&yg3Gasyc&SG0hd9nH?I821-y)#9hszTJAtU?@ zYRq&9eo-nGIC&`~eY9;^=kLhhJINnKGDljq5%#L;oBRzlE#OXdoC*iULb}4W!D>xk zZu17LD|RB_sLzX?t&2(s3_nMJdg6^3I+Q}1#VG$3HG15L-k44bX`HVjIP)GSlf)zTozh`( z<7LSC*u}8`nM4$CK0+XDxcdtpKAE;OP{T?xZJBEczi~efH8Ymp$jr(eP~(V<=6v`KJ|%%&A!`Wod{?`2PKfWS6_IB7dszj zpG+~5XNFt1gf<)FJM&xSTLe-PT55**TG&$Dl(cLYyL)j0EuUE386%-#%jKy?-z}~^ zdf=UcOORh~rrM3*uZ23K^>Sv6g1u=X8J{iZC-}|Dpo&|8<+YcZuYwf|ZPH;!PrJG? zx|Mk7jDc3yX-qk;QK{AOG17gX{M4fcwzRdlx2Twm{8)8 zOLK*(7i4e9Y?&<4DcYy`>dfZc=qO`JC1zhXMb4v?XC|xYL7VLgH>JW;m{Hn_E*} z!7$rcNkz-&=(6y@VtI*4pINv3%D}Ut%k?88z2?tbgqG7ag;iIeuqmnQcI%!@FTd8( zNquN=8_|5BSu!FXgQvR;G$d@_l9_%K`B|Gw^R~I0Q!3Bx&7X)%(Y<%L5&GjiFTYt{ z4qh5TBv&2}4&7q*3Kg;cH|noPQVY&i44Z3f(nc!I_EF~uSV`y_*g4;UP#8GsH&P(6 z?MV0JOla1{6RyPYD`vtx58L0SOX?kT?cX1`fcPg3F^Lumh zO;=N0W`qu(TQqyM(9d}B>+YnrdPkS8Off@Y>}#Qbf@>CxljBaXTpVW*Z0WxY?GPh{@i&RGjVgLs%hG|vG9gvaZvTkOSWEx&A8 z(KVBKzO2%7G(S=nBU{2R+Fvj>bXz)apIm%$3^nP|Sk$uNmj92V5vxr)q(7=(8@%^E zqTzSMV#@{&IW`Y1dqcwb|2%?75(%$Xm)@=SEVOb&Ni7|yVxBLw-XRC45Tg}gfswih zE(HgUzyLNlibvsOn7+Co7WrjyTgXY?KL6Up^vdhOaOi>gQCiXBR6xDXx55{If4acj zzPl<3S&Xg!>FlC&8nk9(Uto(c{KZA`up@2sx3hJqR!xU2z8H}pa4YF#Zs-fV2-tHm zPuBCEa+0w4?MH=-ldf)M17LL-BBIO=5FDRSUUbp8qSQl%PSlaI6M5@YC&BU$@${xj zR@G*H^D@0y)8US(BLoToOEEEIc8VM~&-?0wVYKZhjIm+tm@pLMNA<=TDNkyy@Wj}r z7PzHj(<`^~GX)cieNHjX2ed{F8v4^_8fACUZ2$B#4eWr(MlW``q~>R|I!PNL=?P*o z@ge$E(|+i-ek~djJ)z>9h5c?3{vIN&gwWg$dYGzZ_Hq_u>s;orA;qli&BvHER`*kMUA%ac$@8CcGe`t`W(@1zV5Y)SaTS z0isF_-5)pxCIwc0tkaVm7{ zJwx+yFRj~NyXYF;i};5U$tb*_hS1mdK6!7{fGH+Vz%$WXr1^^om`Ri+;KZ|E=k9&| z1P~V2NKRYA4+K9iqtG+@p7fn37V`e+_XRv>qjZu6=G}>K4BTJqTg1oLo{*_j zP;+iu`C@|6*`Dw~*|U1q%<9=g2{IjC`uJR_m+Z2LKN0NxD*d{@#3#048Bb{5LB%Zm z1zxtb+^ti!@|f)4&`11lH9ZoiNhQ==tNl@_pQxB1o~*)Aa0G{?bAU*SmFa-<69V?G zYmFS8S?hbzx{@BmRN02^wb*?*9sWwIc7lNL4|CY{+Hvc~vT$r8ElL&~S`>S9i}kXC z&iBh*V`P}d_#HU;uJy73N7|e$lt*741*jw^>zJXIMcbjhnW=F_Tw-}Fx0IInmm8HSbwr$F{RCZN?;iMZY4f&38}4OP%cZF@V*kY zlsXfhrXc(b<{dO!j(D2A-K$>*CzP6OSboHEk$lVCFa4b;4!o2#^He{65w1l!g2 z?(-vEpw=5IrE$^MFV*{fk4qVKhMm{TFlcfJ8ZFla_E4fbxY%g zdR7%;ns|xzmG>V1rxpp~!BGk)foDBiwI`T$IvU&Q3W<4dn%}acN@%g-yZC;D1O;~3 z2fQW2Hs^8BvOVjK0R{kcv&VZG2=x8YxUWH=Q|r~%fj|c@{MRpBFrPy2(TR+ndz?6S zEl4+Z>Xf8?x<|8f*>l$(@2260s*QKlhm2*gMTphK$f~6ubj*J_xMQu!qzK#|v?#wW zL>ms@h0<+&YF%S*_xBwPQ9jfJj(zH|mA=1X$$y=J`#s(8x7QgmUU}VaE1nzmM4uNr zCeZyA`rS$|S5{iYz!O~FAlVZOJrN^Av0`0F5!;9!ifog>-gs-_X3T*-{OX5ZSjD(} zb?RAC^K9XR$+n+KW8^Nqhb&3_Ofbm)FB|pywQb{2-TNmVT*8#2zlyRX9-hPL)U%>2Klo@E~a?Nw8b{B6ha)tlhqv;xs-EP z;1$KM;$+2gvU&}H+P;{8mhtGq53g-Bib#zjd;pLK1aNwb0>slSd_?5!Nurz_N_!@$ z_pA#o0!kdVBV|^Pa>rU+OVu$IWZ3B*M?^81C2&atTiPI;EKIK+Z@K!BfIuv@B8bKG z2|w)OD_F0`@NDMG;bAVUAsE6XhtG_xHqnM#jhZB=1a8qiiDGe|kAglNiu*bW5YY5W z|0%Vh#d#i7OyM8_~S7$P@x}g)`!tmg`$A6-#ra zajyYJq-1$}!~`a+pGxOgFgeCnZzdYV4FbIS#o*gXl{-ZddM@Sm%3N)bgz!S`@v6G) z90=7HFH+IgR!bjV;sZnx?>h7d_OQH&OCm8tR5@IOZu(m5QA3nC?eTy=Zvz3)w&sj$ z3GbY$uIe?7PhVBktm*V9t6|EI&Od3`U@YrmFFBwrweFUMudGLx5Llj|A5Iq5Z+ zt<}lzuP8)Oc8?#qyD)A>sYPusF2{#(WSc{3Yr<4jhk7m*!l%hQ)XS5;d#r3Qd7 z4jUUeSQ`QpWgc159$~vQ&i>|X^F86Uk=3f|I=mP=mpefm-V!<6zC$DG<}iN(31tLC zKBgwK?`k*#%mNsILde7bH8wO5IwU|EIJ}DLs~dJ^#VVw1wuRAMQ6*GnDH&)~zJ0BV z4A6*D*)!&#?OMtWlr)h)yjWREIz7D5YNl%{Q$vK?NL~UAhY+BC@=fs8kHg4#@o0*3 zz18}W$onjb(%wejwdj)&i{bl;h$RgEFzQ4#Lt^Mdn5y zXjnP28QB0<9WspJoTQs{Pu zSRv!D0=kl=VJwu3DAc+BCaQGIXL_p|Vk-8yBQdt>HUxn_iCIJe@ zfXKJOpoNcqYU0YQhL@`UBH}?052bVVLP|Bbg77xW-nl$+{c4&hHuphNp!0Y)#sf3( zJ3{AX^l3rjj=cNjWNt|skC5y)xQ*I5i)&lA`@E}3=*;ml{o!&6i!EabmnH!6PeLa+ z=9t1SchM+C<*a&uhbXpKbt{&^im(>d~qU{H905;ja+9kdV?&EoGd zD2AH_FTv!oO6P|3<=$778hq$UdbEQgTN1vn^Mewvk&CwYUbo^E?ktYaB^ zmSlz)&RwVPy64lv7xL8@nFb!c$xvYdL*`a_sc5;X`dN~g&&;~ioxA_wj7iNC8;-@TMX2S)%nvxfJVW~B-3ua$$6)N6o1lUx4*>Lb`>Fb- zBtn2WpGYfNIYgfw>!g)9m-Z@In@C?nFH3kK_c{uj;H@Hap9M4@=#%IMvkn6V_ySA!ZEr^hBJ*dZ3gCvl0mq}}s&{g1Rbtj(!324gd5kG=oXlz@=Y+x8l<>WB8b zd&GmD?|Im?uNaVipVh+tzl{0Ab-n_<{8gp%KA@&{;oFH-_X?{}D?s0AZq)Y$1pjlf z|9LvQ_=6gw8)He!-5-LyI#><&$lmAIaiRT7U^E@^(h+cqi;*8fR&?|XTKolc>-daH zAg$Pe^B(9<pwf$~|+s56szP&byJ30qzZvsKJJ)F!u_dBks>Gw%1IkSpq(Bhtove zywfg_w+gg=`_{VrV2kOR(Ow`S9W-^-f`(rpPZlI2uWsu;tV|rp4H0wWMDL~>!Bp+g zKbqLl&_kod80kEh9jeCQ;IMh?457JcR`dPHj$OD%QUQfxf?$-nah_jaPUi#n6+JM5 zr9}-tr2$U-KzC`=jjP0J^<~#C?0dHIr$?S~qE|3$BUbomsM^u@gOGrFk$voWVm-Du_1jHaCoWRwLky<7ez zll>Hx{@`~07qyHXnIr^34)62s$~Mshh?RVD1A^UX3{*qcyFW*hKe!nXjt&o+xB?+9 zkY$aWJe6&M;vW2K*!pKrEy(aZwVA)MY@iZ`lS#!cq|qy9a$`2WKTri{RueD0y!B9- zOtv$XE=!EPI0XbuTT(`f0e}^grfc@OC0akc_j!q%E47LN<((sut_<86r(z+mKS$bs zl}Q0tTM>EytKmeF-3d5P6dr_>@8L!UEMcNKP*? zQp6(ATqO-Yrm)bLg#cd^#)^Y^QRSnjOhXMQKMA!zyEYNbB}op0bz z9nIhU%EdSNmj^Y77)T)9CypC0b!)f2On_i=#iJFMmeJKTYNTo3iaz zW7A+wuy3{*0-C+o_ozG3|FkB+b z^kC+C=$E3P4f|g*dV4pY@Gp6vT1=eDKcYHgT;1#oWp}|t#M4XEJj}d+?Zrwnat%IO zdRMQ!W=q3|n}$FBowt^#1pbki>0AR);F^Jx+GSs$TsT{l7cT=Bfkq&%He{lhL&LdX zccuthfXVIq_JANjtGVXkm*%){hfWVY0D@TI-8#A5Ac+;~VF5G^dVB1^7$wa$4UC8n zI>`w+h{pX64_??@*@t2}vE~tsAaJV%bpBaZk#hKy^BW=Bk1QUJ(#T0qqz8dfOp=m~ zCq&XqO9bCVztkSSts6B}!KmMXjo#NVhY^&`!=u9|Gd+R=x?VPK{+Mv*~miFhfba)RkTB^aiTf z$`9YtR$4|5ch*^cxK8AAbuxA%ToF1L?-bE-rUZ3!L0%hflG5H2pwxtNBHTl2s6MLQ5IV0dfaIE3fR-_cBGg$aCWQPlm7VN;?ZZFyGw_YIz~dzlh%&tAJdO( z@9CVg7e`H2qboy4mvv~@hW%oDS!}Dr2hGnLk64l}ZVX8axDq zQ?>cNIT_8BK4v!J>wr0UT4kQ7j3IN~ov9J*t*FIHYZ(XZ%brZZw~|QYvOprm;vIr zizmui*{p~J4m1o^WN?kGkRIrzp}JvFIT1$9E4iM$edY5E{Kut%(-BR=c@o766Fu7= zzXp8~zpwA8aW&O@eTbYcYF zDTx8unI2sRvVj59|C~D~cOO@tei_OR1$cPpREO@A4My&BBcaAe-T}?@$$k(jTbGRY zLOTkUo6qj~7PS6br>H@idYy`1d14ap*|Ys2OE-38D41tr~@(YY$h!2*dN$BxmfV0|2xrYp4vMfMjG-LMFO} zIh4hm0HFI4&{J@P1N4qBcO~-(-4?tuuI9DUb9in)h|d11YzNYVQttFwAoG#epk?cg wk8S|jZ32!n#kKBS{+}4Z`marM%_~5SeHzn&MGiP1h=YzFakMQz?0w^Z0n2T7pQ|(%AuFwRGPrqQ^aiqXvwlhthq8A${ zGzpv}Fgcki>ZZZ7g~8Hh;q)SpVHgiSl1|wD&T79so6(#SAZ)4E6*hB)jul$I>pUIj zw6M*kSt7Z*wr2A6!wcN@W!oyo=~#q6W?`u=TG*{3)(tn1Jj7G@erKt2(I|LtI=^x$ z`0GcXp}xJDlz_QdKBLd4%V*PLzE1>*goB7seILXPeFI}zO03RH<V5xNh!T%y8Cp%jju;nWmvG%@h z!@FB+i}aa*!#9~k=&%%2Z)dYMK=_E#euz?wPb$3FQ$8XOV9YSiE# zhDS8){w<@=UxFm)HO#aqRfFwh%VHm`OO-9r`T&d|DNG*y&`sg>kx=aA=^~#4o>!Vh zhA(8}1XYB$zMtzz5~RD(y(fr#+9|~0vJYYh^qPWWB>#h-vKVC~8^C(Q#K#Ohcp26{^DRzvRMKMk}81d9GQQh+E2& zK3CS#dM^40pU3p zCEox@GufMFL-L7ahG@~YX0Mkt^d;G4)6Iu*2&@1uXwci?^Ch!8W+N{1d8^<2xSA|7 z;#f{yxDhzEx8#JaHokxy4a4qq8t}&1_T=jbO_z-APVpz78!4p*6jX;?M*4f?V_5QB z$m2Z$a<6NrjC<7Cny6B3NS>>VkMGhdh_?wJcxFs=Ij?}&ak8j^zL>R9WN*D|wyGrd zpk^{YpZ}n&EZ%=}Vmy5PN6Ic9e2ykhXQr z?es_Xf+|CbTjUpS!k2(l#uEWIRv%Mct>ht4Av3*usDcZe z+Sha{9q~Zh9C^#=KB|-_%`hSE*1^9faA5^QY}KA9iq@|q9H{ZAsRB_t?qOndQ3;)u$PRwX?Ic@!IbN4ozknq(u`J z8}3a^G1xq4mpQRS29ah9tOAw+BrZ&|YW7hW31rNl{ zK<|+vr@f<3tV`oLdox)otKV#uuONosHe4=-FG@uhP=()fkVsT3&{cf+a6FgatZoxx z)swA)BXV4f!~5x*%lufj1oBgJM4Lb|+bMNg9%UPuHg3xmve6$ ze+i_q`q}2*;}DWNt=oe~U*6u~+~{hzonD|ie^|(tsuyNOTsX8|h!@YvpTdo>xH`r@ zy#D&@^0RB>JCp{L>{3yGS|LwT>Zd8iSTP=Y$pgSpOzz)Sk~*A(WCkD{f~Dmb4!s2A zJP1@D@Z1i&e@JQmdjp8q|A#d&HoMP}#Y{-@lk;J*TfTBZG!xn*W!X97?#HUp&bjZz z%sI_7p?bcj04+5=iAU$#Ir4dD;E4i+#ME@GiseZH8YK(mc>%!-NqS|EFTEE83Yn@W z#LU>M+J3IGe8-DBS#rKGygL2I)mKAbo*#YAU7Wm?xJ z9{;%z59|PTmhX>)w}Kv)@eF2%a0~Dr;O|^e;U7OX$M;yt1ne3WAR`OPIf1Yik| zCK33+E0>0S%Nb+v@nvI6)a^z3?0(nMO-sN4J4>z9G-}*2Z!X<_P@*QIbRAc(n*cCl zMFD@z9&lSF#iH9p+V81Np-G^W4g_PULgBr2r6_*e_z^!|^LFSK$Q{vs#OE$5ZDxKW zDCa-ndmQCiYSqU%P-Nuuc=-Jz8_b4k`Tq8N+?=wqvZSM9874j^hCM7S?4PqH81o@n zTU(oqv8@gwrxTP^Q3*pXkLoxs4O^&XX}z;STXeBBHZ}(B?Q9N-y+Gj>qz0I)db0*iY}eaCuQf0_08!kYjgPIC3v*T@M;N>bqkx|r0&#M%g^Zb#OwU=?c?e9CX6 z(rJjGH%ZKgnVr44QOD=F!zv0y0(*m%PRpT72yT(8M05AL`l(hB_OU#9;{MYrH^Gzz zk6T%kD{ik=fKnR78vpdeQd0D7GygaAutBy>{P~Dm7*N#J<-B_JD)-YBCl7)Q!vuuE z-(m88=2~uk`4mmZ_*}P=LKq%zOPb}nI@4rKhTq%aSV;nemhL!zeL;=qibrQm;8_A; z@yN5P+~N?gS0uoy#6P{T1nx)(Oppk^07S+&7r?x3rtoInJ+cISV6S$U|M<**Mw%GT zN7Qj9b3j-vTTSlFK+P@Ju%>yAJb8CGy5bGF-T>RJ-?P6(xUOIC`P{j5yEyCX>HPB} zCr>V}`jit~6Z-wT51wBk*_CZboUA|S?mwAVa{1+?4B)!!Cua5|DVrdy1CsC8KxBu{1xDR6dWn#| zjuVI~J`zJnQXfID5~x;%$M6nJ_Iz#Ko=m`%Ya$T2S$q&UA_fHR z{#lq`F4~D_n@N=mOQY-~!cWHG8pWn>cPPyS{@Dtjv3R>Nu6X9$xY61ftB2X%z!r@o z;hH`qw_C#J_{2N|TKSI!DE;E&n%elNEyk1Z)4BG5I3Ix*m|cUij0OA#QUdz$to>x@ z5#wOG3S#7}sZ{PmK0%4sLZBstCWm~&OA=VKc(y>+=|gxX5(N<;Ngn*tA;Lfv0qF^{ zl)sLj5FWHeDWSC|@3B8TqD8Mlf%&IKhcq1``hTbJ3t|4d5x>n51nZ-2n>v?n>U)-0 zbUnZ)oAD(-N^N26YSND$1%02Kl(Mrcx}kw){=ZN*UV{%rTOS$rq4=1)$f%*|dt*r4 z+`psb1L@7bV`S|nLJE)~Ke z^Qyl`+63X13xj?Rpq@Y|Mp{ND4XArKm$rDA$w-C6Kl!0o=n3@zk^T3wF?_9A^5Wtz zjLpo3=F7{=6*V-X{;5A?zI9Z5Qa9NgF|=05Suw-Jy7_mUTzI`=YIYqLRnLk?=zGH0HQgA$`F_%fD(t9UH` zYXE_#;(hr#L;636$!V`y_2#~=6rtMrru*vcA-H+MeUF1fOWMx4k^U&aBB=uKG#>Mc z@DRP>{Qx09^csdzAGRRm0toq&CE4=`A^8A87M5O|#HT0$2zu3mI>zJGudhK55%Rel zK*+}&51bFR5$Fm)NH1HaldldD(f~w(C$;<@=^he<|I8_%KdS0C`QfJ=SV!nX?n~Rx zu2`;=Iu$568pKrItB^DN`BAis?nF77+Uo@1r!=Vs#J`YDpNI!OxRE9~%M=uslLxWd z`3lJi`4ZINdp`DRXIg@!<9vlU&SligjVWfIT@fd0Ej-}Ngn8+=erKZ6)hV~gU*j6u z)@rc~E8crc?EY${yVGfzOJEbie!K$($#`;hyd=Z2nL9|q%+Ga?A!xOgNy512!uG?r zl2$pyIr^cJ2khc50Y5pvST;Emv)}J+quTl&Ke($^diK(K4U>+-R({0e=CeU*UH5UW zI;a-X330)=`<^N}bv1Z;{+{aB$@Yw+aGpJ=jh3mq_Wn7hPK3U-TeM4jhk068Xdc9-?L?5THPfQYYh^W!`OD%sEsR=4 z&EUy>Ns>=zK(fmF1IsVd@gz}RrXE;H@5x!W&FQ-?@x4F6*L691kp2o;p`%t^TXsrr z0jVYbljm|n6?iTC5lC|L;n~`Ei>o%7lOpN8R8)PLo&Ax89LWm73Bc%lVBjW6D zJFGjOxvOft!>S53+MsbLdzLZ5RXU})Q9xFP*3_s|W(Yc$YYls;fsvkM8WPTim1Rwr zQw?f(Mei?^PPxx+@LIsr2-GSkEIlmWt#SdOVy5m$>+q^2QC1Hb*tE{#kY}|zi)xsd zr_S@HT0oYtL|bM5On1Da;9Za&+n4WTExc`e8SF_IZzBe&V(pnG7D!`mQQNasvg~g} z>U9S=hTX&dU&%^JO0T`%iHl&R)T37gdxgW392YA+5QFMNLf!JN1tl+6)@jM!MB9%# zm1{bG*TN{j=*Z34UvtVl0bp7{ym?rM#*FL5(O_&Ca`J{S+X7{2X>zn8BV}w4- zgC}t4OTT?Q{JeP{Mzxpq81HOxK?yY*->kR9rqHYKhd~45L|yq!OjMp-Cp8T%R|e9gSwaMQU^& zY5DLS?_JBR=o1dfW}B{;E-78@`U;P8Ds?unPD~?+Ag2qaG{5gDp#GasQ^FHp97m3- zydzNM>axHGzp=!QJL1ZG1reb;K7L#=z)c-l4_(Xrb`?}BxR00Qj8-K)x37qW5)CBPhV>2-LP(L=p>GtOJUsJOt^Y}eU5D~rfl=xA#Sota7YU<`%+9F6=>wffjvW373U07IF@1bpm+bhia&!DOnh|P zE+eJw-Am>?hO>lrL^SWOOwcy|@^%6uO-(o|4f(Bn|EZK5tgV^%&IN)9Plt@)Ln;l< zlil;s+Et~Hsy|m`K&U@-6xjcdhWhsy^z*dK6@7hunYp+WeEs}P8iTKvTt7U-pzX%*xN2@;^ zOU+56oqyPjR5Yf`#o$vLxaBb}d@cos!hLGpNb;dSyT^E_&O-=`2-S2P!e#;(*wzVp zK`Hrlkd_PsiHCtg18?mob;r?`J=aAZe>wJNj9~c8wEo+Rx9*v?#dAzfPAV!YGJ%YV z%~>MWKR0zJeO^s$1=8W%Hp6%DY9y`Gu-)p1Q(Te<`+IwEqVXOm#%=N>IED|JH*f;PMV|0;L>F$TLp3xs9I)LeTTt)Q3DT7v3M>UWm1S6W zRmv5nhdMeON*VH{%Nd|Z|IsK&c)DaXR8q{-eR}y24E3U7>w^v`Yi$^!Kn|fz{Pv{U zNzbv|q3WCz+5Jj^cH42hB;=SY8-I6Tpn;IE^-LsQjp*9gk+lVN)A+3O@Y9L6!?qjG z`^Uw{y@=@xYd++GO6n!>R>1pBrK+8FVC;N930-|}{;gfHQa51Uz zVUl)kF4a9god*F`G1e6`=e5V$m#TMc%2z&|noyM<`za67=I?j)#U>p8+RMNgsS#ca}D%p9d5lToQJDudJdn zFlJR@-jUodrY-@s@B-%A2xJvw@MZ;0tz5@rZv&rl9-QWPWMFY4AV2*A{}|@!GZ(;W z<8W0Gpuko(AA=iop~4qsz>tUzaq6C-h7&|pBs{%VPevU28e;_z3ygEq@hEJ(&*W8G zfALz5K%i2CD)35Jqt0|-^%_+I}l zK$otrDRF!|;?t1{E4>Ls&r77bU7Qa=X5bVxXNix=6gE!FY~}$&*AuX!&2F0`5>LW# z;jgET^G0%uXo`WtTr@UBqv~nVh&R3<`3xi2-Rn;puSR?U=E*3Sp@4cI5kS9aY_cSZ zIEYf4hFoB}0;f5kSy&;C3tVFB^+oAZ_iv20NM9RP<;^ujE-;OWNdUJ;kLz$Ret$r@_c#N zhO#p#a83hGwjpV_{LzKDkg*#ahDkQUCj$U{f;)Qq%ay3oh^^-K~p)!zxpe~=w&M^jWE@9n& zB`pfTw&~X20YTn9wueeeSGJ3d8pzAnJ7r9EfBuk%an|hGUjljQ#{+0_@Fvey@Jirn zj^19=KpP&A$5o)s+b@(RHkkb-zy^I_PQ<7!={e9CZ_IW{fPw-OwJRbbdSx~v*FjZ5 zao;95iZCu4!JzQRCo4J>oLq#=U`z3(b0Bo0A#n0ctLwriv`&rKH*?@}aOPC!&woi2 z^#;jm#?E)PW8LSUYlDg{Bfwf)BGc28NsP-pqr7`69IVQLn0b|-zmI^8jV)+y&a(Vq zeeWvA9#sg`)bFKzy2Y&Mi16^<96xdf;h*E) zs~}|qitt?(ZAI{DJ}+ZHoPr;HyGR5K)KjRo(|2L$odRC(1$m^h`JQrw*%o*cB~C!S z0t5U|@zJ9*b1q6xi73ql2wswCVdL4ac*0MS%+Ti6>`roVyZ(GKJr|+l7&Q#a2+%eo zf;&ntEAE0|-%JFr8hfe{5Z)*-qy zIC<)n33y+I8-;uPr{hZ!jIZ61=|e$99sTdT(V_QLfy}%!)dZN?Z@m)g1>B7g>iFlN z)dlWW_4hmcuF3n|9zyUhQ~7-hNl=~`26;7d=#GD#{O?Q5K;M*xe6l$jvNi|2srYLH z4(D_hR0mvz81f&Hq;XI))0}g$;m@4@%3AT%4s4<4q{^F#qgo0tB4ECxp8bCk%}=QS oE4=>s$I+brl|K6)T!sS%s(U?cc8eAGH-#Zm;_~yaB&5z?2Y(6EK7qrZK{+^_oSfL5xY_OOEjTy@1Ozzla&d5R zv4Ik7Fc(|6ku#eujP7?O7xhS(!A$HepTRBdY-v#S8X4O;z=dgPQ4Rh1_uEdmrTITv zvW1;@3v`eJ^$iCn`(2Jdb%R%hP*0)C_LgR#WmNqloI+wmih269bS;{Ic2^wkMGxY}TJ(@&1DOPGW{LPW3C?lX9CT;WQ( z9qFf%{hGU6&W{)Cg=BcN(rUAs#@O?BTKS~ff!wS%fo91niFw+TSmi^;#XepELL?j4 znqCs=p z`XSbbsU)v0OY-vUFVqi1Oq`-BLP9Y#bWHHWn;wtQkbyk>`^5uv2y4OFH>e*pZ4gXN zS?u(CsG_I`UTA?tB2iZzIsufrs+GydA;bEVXR_a5vC6V#4pN&X0b0? z`MBp5M3vfcUvbe|9D%5>+naRba^9GJxvyTJ%Y1|1HauH3 zGsa|0B~sW0%4SXZ{dm0Asc_tB3ei_?rBOYf;PzdZm0{mwuXd^}>Hdou>XUEODk>tc zXi>A~%+iDtSve^yI)*_YM58S9Q{uy{y1IHh9!DLG!SRyDgo%z$q>fk^;7pr zm-+Pixlw{1(bKvojc?C}*oWioz_mQh)Q9&edU1@MPL9_-m8(WuLa3vd65UnUt7nK7 zdsF?~+^yQ}39e3mh-A}I$Ui1D+*uh>WpDUp?6Q=dIWsnOQW57!ak@&}Bp+~x4p>%z zb{bGi5?z{9iOu`2X0x4zJG1u@+u&F-wdYZ>shL?D=-7>Ne&O6vYC*>p!d2?CbtqPZ z6$bSgBn@cxBj}($*|?t;of=_yI##Z7ui(lkjRx4WLMd&L`cP_(EJLQ0_N_mBQ-?(qv;-))+EJa)NV)~e@CS4YdkVq;mzbf0+#-QZUmt+ZPw`kWK8 z7Bp@fm0DT*>Z9hD-rQjWPrZZfMY?*|9eY?@PM$Mj0zya1^Uh^1+M?6FFj?HM-CE{Cg?X2S(~Df4Wstl>t@zR}BVT{EnYA3tsdRi|ZT$v_>JX4V4semO4}U=0rq zwX+n~(Jl|=BbSz*GPAQsUM6OIr@Z&E+h-`r4hqOEjDT4%ZWZo9uf_PZYFuY0J1U&jYKutU`s)cE0|M;qdm6oF}uk|mcg7oh7vW zMcmx9eMr7Ad+KP8f3toa(Npic+Hi7w7;AN!8Y04Vif&Z7<&Q!(LRR!COwEIq9l@#n z;JD|X$xEz9xLkGyHM(#@j(=Ec5!}3IDD}L?Y2B7Tx=H#)?$MEZ_5NJ!Y#gkc^Zii2 zvMAeupPV!goP;hkjn3%6?_tLgu$LWixO+XGiOCcUsO$@KIYO1DboEhxigoj-3=WVZ_AlJ&1T)PdQ*sP9!Sc@l^Qn6bWmojq!cW$ z3I@=`hb6xS48aqr82DoqX-?i>kMJm00~AKdw0%W(GT(1)&UQ$46c4|~wuoU=j4_qM z;w9|s=>EM#_<_IZ5adoo7XA8nCfslKzQ zM48NR3xz0{s!jZ`qV1vBN!nf=a}C|+uX+c(7@&)6$*t3Jd&>*287KG+o6umniZ96O zS2JW!X&7UyqqTh_(<|4vNx$(9>-Jke`dM+`PDkH$FKd5C9&tPU|mSMLlz`N0#_WmvQbqeQ28+pB9lH4BKs&ClHOp}&idoU@n z?obK~B&FU!FB_Y3Vfwdbzd-B|(W7q!HhjG`mK%AMq`c26m?wV_XCM0F-%$AZqa}pD z5|dY^OxbqynvR>t@xUp;lagao^1B`O{NP`4&|} zp`#h8LC`77fviW2U-}HW?5QR*6Fr{#u-2PdN)MC;Ifp2xtyNH3BqA#EGhFOTtFK9q zaX>@fo}JBvx~RnqEuP3&sOER9^;p6YgO|_|?1msG>`rl7a0jfZjt6vPdu5r#tOD3e zVe{R+S`$0YV}|p=BN#Tq4kGIE%<95A?bKF@Hc`QAA7izDs=zC;vv9YkT_{= zTgZHBm3JK4P|WDHj1x82OCvAsw}KP8`z@SVRYJOtXJqU}uQF=u<<&@PORZIQ6c>!O znyi$plEq(9etm#Ja#i6+SI;XYI8k1LMH18u{*A!Yb9 zw0ry-840QRvZu~Vs=<6Cu2Pp^HK&hv%FI3hA{2vQLxW5BG!?&frp|cG|wOn(Jux`1rBS_1&xh5N|dt} z98uXJFGZN?J=rk+O#}v8Z&`ED9vcq3HmwIA9-y(Q>kPq zycI{5wxNRRfG19=ufpJXGgI`qt&a5lS|zN>ntEil?bpDxgkAFz27%LrLlm?9e0Thu z>Y<+9Ps@jYxpZ)D*0E6*?h|F#B0{ZZ;_e_ms8K6@DO~hdsxa6#BKUnuKErx&&2aHX z5igro$;Tk=h1S>DYzJ3a6hmPND>zXN6!)d=;ghO)0&399di{Or@?HJ5F&fem@cR74 z^eBBUIu>nZ#ClW%ovi9iF7fyLOv+cKu+K_r{DOX6-S`2WQ@UFIzoa5FJD|xrV<~jp z74bEC^MaPHKR=PZ@otDQ=}8ba*Ly*5Y}+8w&*AYCkGN_9={9Phoe|G!M;-KKTHUUE z>4;@a-z~v8c(~4gK+o>Y(>^QJ7ee?++bjV*BLdduvmZGG2{^dE29Nik4kRzJyOQPv^q zL(R%NtHqfI1ERYJ!TGIep08J_O(v-9z#@+lk>z(<(*;0fS6v)#!hfm zfSECTzk#r=uXpZO4cAwyT0_;kO+ z;G|?9mj=Q1;zWWGVXum8s1R4(UcEm-+VeApBeZ6jxHwm%kdNc@AX+zLy>Kqa<2Op9 z95LH%YJX;o(3;IC7z8(zthzPO{PTapjwv0)3sLeWE86x`ENMeE2-PB542a#yX`bP!`hZoo)u45`a$><5e zS0|pgP_EfjY0mOiDoRr}{(NKi&ZCb;5mDZ=ZFA~K(mk2*CSkh3>@%!xPCjkTH69R` z<{+gd2`5C%BjF6?L@Jn4Th5-5_&|JbezngLn-83 zGPGY|#{Xy!lNXTI7;Bf%u!64R4fS!UdoC?FZyFp|I3vVFPMWos@vskcFR zod0@fQ^1pr0s`}ey+;*h9tPNby`}&bnxY^13Y4S{X@O2O;tWtD51JD{sm12i8HeUMKA-I>Q)pS1 z%-I!e{4OJ`+!NMc^1W4t6pZ9F32+fJ(Tw>ai$wP-HdV3?)$T+Yk~?CtcTbV3I$vDZr44oFmC-k`eTamA^%JAO`i@^v7T`Z4tcG-4`Fcy- z_lkg8h1B8U_TrBMY4+M1{Z)i^&(hTt=?}*WeV-{3>#~rD0$hBOpaafj+RsoI^ zWof~fLu1BZu0}1LW}@dwwfk1=RCrhzQ8brXfIGlOMrHX%TInadt7A+Choi$YVe;&u zz)y*RjhX;DNmDuD7|UFnm#1)*O6ZAGv^zMKBdiMRhbHEtUO7|7=GuKl#8~lVagMG}xPPO4N0kjhL%juiw*8ZJz6lrkgrmb20<&h5hl)a7UrXPOj7; z9xm<-ivN{$5wQG(epVg6d+@Q}c9Z7qWiV0EUpmUJp-WGrps()d=Nq3NV6ajOm!J=U z^VJrT2mGgz60<0g9C0B&}^kt z6X2BH0sd?O%Ahs{PCBaaPtDO-B_ES*jFr^q7ReJ zWdz>DkSGI>>gwan`vV6%%d^3PYj5|aJPo$h<0nZ6j$M~Gj<$ND#>aJG1Ksh0iqFzm zDCj{?mnAHJB335rb!y;?#$06Auk%8)a&qLlq06i=!HrJtp78eecB#Xjdq0AtmOZrm zTR10ce0!NDE06@h&F^U27_)c4w^OGEjwdAijqC8Z{(+~EV7_VKu3N+FfX! zQ0cXpFAf;#Ta=32=V06#B8p&Ir)y41Z~Cz{=V%~-U{x_ZWG zfd*D~nb0u(O#(4W<3RBiP1G74EiF4^kf4e_Mj&PJxhe&neT@b&x(){1s;7_ryakz9 zAcT!B2ymj_24!BVd7-i73Cq|1?=wI~)~noUT_0!yw#K_dI(BVfs2D^=4SWfx*<<-^ z?6+3F7K@jZlu#UhjlaQX{T4u4TSo^HC=y!bPa`H6*j8cO#lO1?40gW#b{oxVYp#pS zyz{T@tgI(}LCmTdtpHL@0Wah-`hXeDXA=`^v9mO2t^?$*uhg}$-IWn8ho#5aIXPiK zVQ7yNfUU4wS|-N&V_l4Ulejl4)4OctxqVzbyjj4Qv%@1IqQ$bSO9&{T^ulht;V&)` z#IOSCtT4`T)N&Jk7z541Dl7r_yGYZG&%)1A@WtL)j?*`X^d)};wR zJSs0QfBv_D`$Aam?EFA@5F9C;LV3VTt+gn~%Y#SxU8TI!5y=J{M&U8gvBHo@ItK>_ z^RT*!kv@^w7LO%~gVwFDtL|r#AF5;ZZhMg_DJf}U&QyjrQw@4^-SI47N~|i-fo{jI zd}Z{CtIk8Fs=dYj7I!I;nrPA|6K31+zSo_|4`PWRMXxR$&|`do<)|MD6JxaW@W6h zkX64<6C0lb-i~0zXguEsXy_qWDx`)k<5!jZ&bJcIFR7+}otF)kyw~qv^g-n|*lrJW z1EhXG@a`m`X=?BE$Q(T1zi%Ia{Y&y)h39-0O*6puGv5`=zj%QJ7wl)ksFZU^py3Al znX7cllK1?Dw1;4SzN#X5b8bX5GiWgSSSk17g;xMDm_MdJIbX@G3ZTKx6({YB7np%X za<|weF95?FG-zht*#Gnb5R!pKxX+m>&XsX4RnQ>!VZ8dq3%I}{mDlpU7d?tboy{Zb zWfw2}=R@GVLK3NX@jl_Ljop@(nJ5m*NWDu-bQRbp#nSVv)yp+P<8R z%w3({K5HKAC=_E6E34*a@+g&xyKR zq{61Ke_`r#U@H0WEYDx(eN$lrgZI>^{haH12N6#*e+XK(rvPZ#3lSQ*&h`NcxSpEV zwY&;kdm$i3>3cENLZ)8vprKCz;N35f&msg6#74Nyl%{8f4Ms5gzg z)S@R6sQM_0I*3wG9nh~IbF!aADxZ=5S>6O#VN`~Cz>na9&2_$0)GRiB1t3orP$bjO zLF6SK$FBSacXq2^mHrKR&w=c-wNLONTj6kNY3sDJ(H@-CP-TYJ(UrAdBPY_ zAu}38*9v_hZRT-rKbpK9-V)}X2;$&fPb(9?YB7g^#Czs+j~*3)-S<~Qn-j4z`zkVA zKK0o)iHA9SzE{d{#h~GErV|OI+t@ZH)TR_O2Y`N6UEZ2bIyf8)k5tCNp}aE@9B249 z=6>6AMeE+~H%io^^^8mUGH@zkfOXtq^+Nyj%6{XOE)@Q$Fiu-~@tU=hXjMH+URZ6) zOmLNJ*sEW!#+OHRNI@%Wf@&oayZ+?;Q0DI0YL+FD8DaZyaB#plc$3TYwe?6zdYj|K z+W-Wo@<*wQQ;g;lBq7)^y^4(_%H^It|IX$T*9kFLJAef(va+8XgR zL~H@T)`O`}JcQ?F*8%3asU5dt@$U0D?}nA@0I?3aC@;o9fUz_ zRe77VOHBRYtD|aCL4VyEtFVFXsXc!DZnY-4sOVvPN5_xRd2{1`0@3Li`r9E&+Io6y zAdLCadc3N*mmJgq1oa>)VeM_lZ-8RWf)EGGW^=*0W_1j{{}GL?kdUaFrd_QB-}_)Ni~z zF7nvs6&*;iarlxcq3Bv&V}rFn{iKpjBuoBIPSZ{#?%+@q`R;Q54wr~vnCF=L(5>v; z;CT?7xsHc%>8f>K3W#h4Drur(Unm!oQ7LX=j$FV#ts0S4NjCrQNg8^ObP?R=0F2{Q znm-;HI}h;`zs)E;5E_UOZ(b;>W+;TpA{pBNolmoyYAA93X{kNHF+Cr0;PBvzs zjL@!iw8_)sGHoRUT7dkALu0C@{{V(lvS_Z-{Az5;CfFqB$$ z>AXZAJ0J+V>`CXyMMKDka=&(NubvY=?=p~rkY10Mbpfw06~SR|EiC7D0k5k7UM<%l zcP>s%BS2Vibwc{6F6h>On*hf8zr;j%WugxSQwq|vu;liJT>n~R^ddYwTt;6%Av`j& zJx47U_=u|_m1uthDfL2qHeyDFw+9EVdo2|X|JJqqH?IxlX_-i68kgIB3b?MKuFj}e z0}mzVHg5%T(u8&X#5`%!KRJ3{^XNli4DxAK!v$Q%A8|%1Y+}a-0s?SL_4r9hN#$Hz zTw?dCv5GF%7|JJJ{Z=g@T|~;NNdr<2NbB|P!R8cPPWI;&uLHEA7pzT&R##W6=;$n~ z=a(OjLzGw-ZWaNwU+7BKYP^TZ)@zk^D+05DmuZaXSH?$cv|$~E$TFKNhdYscU(*ub zd5(oC#e9Ug?X9s%N=nKy10u!3RYeYkXR+#3blOPVzwdohz+S@M-af6MKxuPpYpkw^ zZTh263IiT{jkRbVHKL24{D}_bHu@!`eu1sdeUW>kHh~07VF7oZ)u%rw-!bz&cD1Sx z>It5i;!AXpS<3mb$o8St{HCbKxDe-)%0-*|@zVveq8BK}3%c;n&<~ETrK!fog_k>^ z_wDTwEM_3B!d6u}_@<>TwYr8z$I1Lclz)OgEXVB9+t#tp=lVV#@x-zK2O>Ho+m z0FE>1y39rnVnj3Z^Bo)Cn$7w%6bokmpwLA1|0LT&os(c~wj~I=nSOYAnagIBr!pHA z0UTReUjB6QHU3{cKocif`NMGq*7J1%gehES3d6%Gmbg0_Oe?#49$I??ZF|e;Z-7yDoBctT-sHm3mzlmC-kUW%}wFUwOmD#|c15|9Cttpz@qUP`n zy9CGb=Fc2Q8{NOz&m+M`mGDrx+;#%WO<;BO>*rcD3k%D{EGqY@#H?d{$HxaEU_bME zW@bi4PcJ^2-p;9_tY@w`@r-IYx@n4VT)i*O^Tk$%!P~B@tpc-wne#@|4k?ns=0>we zeL6nhBGcSdMH-T^xCuzr@{b-pQjyyhY8q`o)ON-TY5<>m*cv04TGTe-uQKxOiX^3H z>;0J_`>gEj6}6pg&2xaEDzV5UM>N4hP3hhAlQUOUi~=*5`nrJq!?ayk=)n(K zm(0K!sOYM_PJx0|h*kCDUD_{sfIpV2JW;fKQmhAW8=`Qdi>FH#b&l) z5&^z}ahwn7YY{*0Fu9J0iql4T5@)++CdaJ^g-&V&Y}ag?Rv*T#f6musw|}D1Tw%+g zl+Ymwg^Nb}r3aA#0r8hS8XAD}rg;cdr5yIA^)nXjSOQu0HRt}I7D=D&(5F|(qG?ILkXyfM>b!>1iz3MFF6GgZ|Ks7%_`8HxR-LrY{$s# z8(T;$zazGezol|7Em=5Uxcg~Yzh%1;nv-FzOqXq_QoUsGpulpX|4m1!Lii~s(=WPP zgyP`f!p4m~JC)pNpRE+N?heO4N*9HusteH0kF(nMuHw^4Q5dc|`6Lo_!#$`-M}FoY z3m>Ni!aD9&sxAiyT00(o@GZyCk-VzThp9>Ngc9DJ?VZ3lq4a>p*UNXF{`ZM@#Qd;U z!?fp6iRMQTg#*kd(}N!h%g5%$)X`dvGhDI`g~i*{OWjMA8>FqBvT7Y0{Xu&jD&QS+ zauYu1vAEA`DVC0|k=cMRim$W(MvNGQ$79|Wo%w}L{1D!5(_L}F8#$`e3dZZBg_eUg z_ueaMSfo^E#5D4cclS?&p?A1rdt2IbUXV_6HBNTzIyDcbrYoKb1{;1=jh;IfwLH#8 zAh85^fA}5bNyNG7sNeGJEBU9s@Zs`rPnLRw&;_&(d)}wB%|`ZAJ3P@H7%)H z5FN`|V0K(C>m1K}t>fR@QWWe(C+79Z?2MRB8=xaxGP_^(-OsLHme>{GAb&&kxsS|q zO>gRp?-A`h;<50JN!h`eA1Mq_=XKiA2e8zc9Rhq3RA>l zu;7zwA@93ys~9jmKqJ@_-&;bSup)-bDUEI zgZQIcVz)JXtoqhFs7Knjn>Fk5!~Sr%&pjdt8{k>xUd?A=F>fBc1*a>d1`H+3*z;zo zTXAFoP;_<02x1`t?{I9*LlKhw!)1Kp(VmkU;v?PuyX5-06EQ!wvp=<_!D^sTbe#s6 z{6pzBxP+o~VTqvBoH4mKBiLYyGGBiW1dDffcc8?fXLGd4Y!IytqI)#nwuhuP?A7V`*A3d87qjc({=Z&hGxJmmGn*?zSo{qiVH zzuV?PeBI=7iEm+x|BhMDL3#)DyP2w0aEGIQ*R!E2Ul(2NgkVjFXN=thadYliJvj4$ z>GAuT2Pvuu-j5>&mT^4^QOcM%fpW)=akh<4@fKFa5N%l=_}!sPzD|0J%jBDJ8jLgd z3<6==+@9S;!b%vKrJs3@ByzT6r|Tpq_x%+NC|1YA;2ttyfj^lG2nt0i{${mlI%3l4 z@bA?p^kLd=y0r5&9FS{}?g+W_7$(a9Lxn-Oc$qJvmW0m$Nv*na&T;?qOn+~JB0lS{ zGz<(3VNp>`>FMbuPX}}8i67nka{#>)o5sldSK&!k!E5}tkzi4UVL_-3^Z(9pCim}P zoFz@62=-ek@1{$d*RNmiE6?4sSysC9TkJ0FVl;XXV9jqQ>)WgC-(FTMyL|VB zypQvjcb+Q~)<}wU2$hPK7Bje1V*hKy2o*KFdG8sGIno2dCz`EMhmD_kN@HB z7MYAjr$-g{l1dlpA}qV9LU^M|iIlP|jHpzWwfMMf=fL0+Y&h z;nvo5TQxPc&CH05gtjUikqz93hX1i{kivGWKHI`(!O+(zIb zxa?hlvyk=P^JLyLz(dNz|5b3x)4PRq83#(j^`tvB0B=$w1S*SsY#;R7^7t?$jE^vr$O zk8ASzC!JGAM1%a}0_&n#ik|m+&{SkeW}6*NmlYhJDsYp$ff)IiQd zlLa}b51o&L1+RcCkp#r<$h$7*mEsoTlSe_((Qj~g=#o;gp?h-knv$6Uu({R<4Si-1 z1Dfw$=Sc}nI^Ny6lH}8&P2BRNWEqM%* zVcxlD`iCp(W71;T!(g+hjJxQW0+`Gx$ZM~u)v38Up}_j`ftP@0^1|n7{s;3Ki$b}r zE*uB56Ho*5$vPJR1Y}22eLiKl5~2cG5ye|YJYR;d`2j6^&M>_^>ceyU_`VHrN1+uY z95I97xXRGPzQNzk)~^3z z_nA_qJfgd* zK1!36iW54FY`C-xwoC+AvdXCYF#8kh9!wxeZr4nHj|YjhZMiUJ*EoTxL8kAVEOflx z-&gCj&Pd-oY=pdGjZ+3TYEo@T`*}FmiyEs+a-l^(S?PQy6Fvb7gqRa`#Dzv73IfFY z#Tr?}=X)Jf8xXlK6VkI6St}vnZ1Y&TzI1U<>wNb;ayf^ZjZ~mOP`kZ?UW8hb02Dwb z5RWf(*dq`$pZ$N>gx4e}ES|a30fty?6r5~*Qa%3H#AqVL)YT)uCW1%`N;#|B>J}_6 z;Tv%TseAk*r1keZf|(FUKU=tL$L}qEElx$<3fauJL&$@Cy#*=?JApNH;Jb@Sf#{mq zW2b&iOrKj14GcIAO6{gZszH978Sw9(s~v$Piq|X#LeVE6Crb-Na_Iz})=1d+L9n6W z^mx72@9D7&$}AX|#ICj?N{}(c{x&g@qjJil7ToQra?q-_PuB&h$bBHYY7qn%gbo%` z%@Zjfl_UazD-@5tO3Zb4g_)z_SBhI5yU@>%#C`Q{PHi9-f=aCG6{#(8T-Aoz3Wr4g z1c`ep2>^_0$AU^00}?nkGxcg68GAs`U^ek&F%hM1GRg=&oAz>J1+gST4{z4_#t*-G ziZ=GR*(7N0LMZaSdPZKQ7GVWg!s~DfZnGJRT)KMm_UVqnDFcXbL`|KZ?Ae1V1#U6Ub-9P zUe1(suydp9ot_-iImITm5B0D8T0cFp6L4I4chur};%zST@l2k-nyF_d zoTefYLH;8jOCqq@9_*dBPItS+X`VnR=p{2RQ8_nZvi6QSsMH2O7!hOQYFYw!*FY@k z(Ka`iWdcZ>kpoic$W0do(*}@K)GE9;PM52mPej?(hh_y*Q&9Kv-nA^Y#$bD(uE?QM zjqGQ!~ejAL7I9SZ )o*R@2d8*^P|Hdd+H3DOot?9-0UxTlwZk%~we}<#~!IW!bz} zf9!-x%mmqeR<^=VeL)s!U~tn&nz!Lh%|tf{e7{8Hg~T1p%w~_InNT6qOY+cAa}VWqp0g`w;Y#;6oDFKKIgi~C)^mR z1rioZB&Yx%;DHG~w38i8bLgK|#t;U~%QOKEInG_!W625ezV_u>;_30!DewJ={B4l8 zG?Qy^oD*%Z1AevJ@+DGqE~t9O@DRG8E&+l&N^fC3!o$|`GsBAf6Q*;P5m8aGndA=L zLm=K3OOi}H=RoUb`Yb)VEZ8+rWo%>F;B>U}$5Z>eH^AkKI!O087K-;x-$0)?&_Qf` zK8o`=Slk)bceAPICYl-Qsm9k>@AS(73XgLAmT6*}g%h|C*0s}(^fPw=$xl1osxNoC zvs91eIqFSIdD;ta?C2JU!Nv+F$)j_eq)L_q;R_Gf$02!zAm^%d^aqewtlEnlD#qe< zqIdYmpNZFRkr*F?lvM3TAOh}bg17_C5oDYee!@$Cj-+UiHWJ|T{Vx#b|DUhOPM@N` Y!kRL@Z$UeNxicmRKL?=lO>qtvX3L%u_%#@@ea&Cr|k|H@~<*?;kIZV!{aYaIl zImDb7nK^FS#>{N{e%94>-{0%HulxJY{rmm?_rlevv}J9Dfh<(_=bgc zyqQs#2V}76^~OjH*iR zKYs_xZnGE;AUaV-gJ;!46LUAjravkw(qMg4tE$9rQ!QVJ9Q${+{uWSf89FV+ zFl-dVW9Cs)Jc@jRNTHPoM!VLcr$sJ4#LhgqWxd6z?+;TCyBao^9do{fqnhkDOPMID zqnofksbZ&^N0`$+T)#CS&*A|Q>tpVi&tsWsqKMu1vve{fw{HMUK;9+*ue0t~Gq;O!MmX#|Q;E)36~FXN5~c()fiH-@9Yq zU|HqYp%_CdR88CMlN3yb&JJk~sn+OGwm=Vea<8Wsm|7-z*G%u%QGHIdvOu*slwObF z^k8@sd3dzid@OP`t9GGLlv@J2*KS+97INwUV_e)UyReUO2D)a-q&(6JiX(^D8 zbwo|4gq;t?VUKgz$9(x$>{N#6wDIO>)k!>h>q}lFbTuP`(_S-B>dqmK&sUQAN-g1L z`DQ3q?RivqZlUeFMnWQGJ!tAdc<;!_JcT=D1bzELqrr~9h(!kU2(hwqDR+@EcLVh3nHYd%m!zXAYz^1wGmI7k#9o^8> z5y@TT)tn~^?62>8!YS1V8cu7;eST_c0XjqoTRN)cGk!LRO-8k#9TF#w zZzL$Cyf*x(M`b6hlTq+koav>GM}wCl2}LzC2cc%@ zygK3I1GTCQvaUxw1?;NIl2eW$mJRx|$@;nYSxHJ}v<96JgR=#73JZBmcjttmSB>Ob z*BfDExB4#j+Squ;U2}bl+Q?>L_>RuzQ9xJ`BPZG(If&Ozk+Xkk-JQB?+9Q)<91bXt&RU%d zhCs&bN#5#L1}G=MnmSl+bd!d1ibhhruOB{cjXM{K2`%NYn?-z$8*D1k= z?{F4$qk<>mjaKV|bC6?=5n&LOK%Yie?>UcyOn>-JVxa>Cc{qN$rcB6pyzVm+F#Dn! zXxkl4hxZ9MIn!h{71>8q?7fW1c1Djz^~6@#ui49L4iyECo)eYd^>o|!^-NrNIGi^J zkHc*~Vzy**)&LXfxr;3M)YqwFT@Rx6Eu4m9ePZqGaF0gA4X-rsXgBF^(X`8yiPW78Rr zYxyU-+{kIWAajEQ1JhW1gD-d3k1${E+1ctRUJ2!|GPoWpEHL8DwG?@J&DU_zRev~F ze`BKC1H+H%r;^i@d%L?W(Y(1*!pgL-E@ml}wEB?{J*2%oV79Pwg)F0H`^p{OMZD?i z+%v_j8V+H!UuIosL+B;PQ#OFsIb9O9E^mEwxA*O3GP!9?lKbTKfag8b~B+vV+%~7aDRqUTQG18eOex70l^GMGuKc&ubPYo0smk zjy32x?|o@ayI_eUp7pr4sj-EJK*Ck)h?h^SQud2;)A5I&$beNI`7Yat>-6o7Tu-hiyFZl- zP1l?}9%CLGJ;#0yOB#Pj#0`#_g0`I#IGdE__H7RZJS}#$U-W`{F!*U4iRtF5jF7uV zJ7cSP@Vsc`t$ii38C%BcW!3Gi4m%a${a-5PU?S@kmgr>_70G!UAuyv{{8+=TIt|NT z5^hW!@7l3_-N3r7tfQ@0@V8is$~B$`4*HJ1sG6@BS`C2s=c2@Rz&oy_B<1Oehek}7 z7#;Kq?rg1!82T8xVZ5G0WiT2EDvi4-FORw#CO=h1h%GW=qI797aIeR;gFMNV7X)Tw5DUB;qYPZM>Fqh!*6wx-pe*$vp_&%!#-+=TUpMJtP{KA1ZQlb(SoQ zKfJ;1GJjJ^P~IT7r3?Q^$8~HkpKc#PDMCpDky16Jbp#jh(^J{oc(*-Wo5&CGy>X~* z?DTF-f6~y$`JIw0@8m+w05N?PP@fZZ?Ch6!L{5#K0&RP}V(?xB^R=78{17jh;5IOo zH#K_qTMXURkacK^vZW(6A?7^u!bSOk4Q(x5ZtNCJ@et@o$gSke1RF<3^>zvrNtFCh!>`&CGRl+A|rdKv_+bt3qZ?Y%a6v92QxC$b} z8^EQ=7ZVHaRzR@8Ja+YDzy4H0KicZtSnRI4!syFHiDAv|J}bT6{l{=65{a$jo0Sf#M58e@u_Wo4!8E60`RM&Q3`>H305@ij<@lC`4e%l zW%K;{FBr5ah$$=hI)PC!p`AS;`E12uOil@Au|=n$2;1rx*OUypCw2g6K0qp2E^9!b z?5E*EAW+;95g8Cj@6&(q1^4n{TGUR{tp>>kb^2Y^mG;q3_Tbm*?Koho2YrsbYJozy z+{tV$c^tIbJ3G=~cGFbKP6pJi_kQh)ic{u{lDevO!5z+AX)M~Y$6R|}3Q-O1RC?>2 zk^ybd1*JR)7h+zj(d#v}r`9pdgXnF_vu(#0Dc!F`t#HTAAj3jI$ApDMr}F&ozeNR$ zpIXqXyweFU{xU_WBLy?3S_o6>Jnd zale@_7{^SQotyKo91YG-Jq7oLqt$@2m{@(>qc17%wwlwcnH+RB*464y_E5O{bxlyW zu`(MEJ;%|;21Rn+!;j*!NDXyFlcnpIwYBo??{7PL z{i#E>Z%)H1kknS6IbZU0$-dQT4UPw-hV!7@X}}IeIXO8Yp_iuCUIs7jgYPWot*(G> zh>OT{T1*Q*^7r?z!m&n&PVbl8IGP!u@_6u4x#FV6NnMO*TRj}z*i+5b_^ef3M_ zSoWFFZ!NJbB6$_V=+GvF0xB;Nu|P#01hg8-D3p$M#-UcWzHiZmv|B<~fj&c(mH`L) zLe|5DTU-2$73P#1^Gq6BO0X*36 zA#b?_S_AY>umkmcE)7L7xU9Na&^JP%#!J-J(D^z>2cQBsLe_Z!5k8TYJB2PiCi+n0 z!eYW%9P#2T0V{ED@|DXPZj#`m4?wHM{RjyWF&E$XDPSN@biiW|>49_|YZSpdm9=HB zVHGxPF`pL1+rTV_u*HU{g5kzH}ldbEmo5y%VJN? z9AK@?VrLCfyt>{=)P0K4S zRr7X=$L-qzg871L%d^`vr+_g>eO&PSlT)uA4A;RK%xe})$I8pfmhnLI;sdmvMwckS znaPa!BHAaeJ@+;D{LAr*AusJpAc<<4qh(PPF*(i8WM2rfy0o-MZ0E_ce~94G`RX)P z&^9|%dZk0p-d!GxL{V;6latB#=Br)cq0ewY*3jLH;pM&74s1roSwT)FN^ers9S#Tz z;^0dvjhA$f`=KkYHj5ya6E^E_*nQS0Sl`|fY@1~m<71v76{ zdJY^x+9k)kr9vb?pjY5*=i-UPkV|>yi-<(8s}6g>OW{`M;&ViWbbx8sr31FxE$@CmYh$bHWKk4{^u;(U9ekW>L(>(-V|~~{ zupl;s(;{2YD^cB;iZUvKc3;Z8D&>5#S$eeUoW*+3uks?+k0vneOgxllj7lv}$3wq2 zn7B<|!JYGbA~HVNTSuTi*WmYWpww7fsoSQ=N2%5Mm!Rips;q^`bbMi-9|&|PGW_}O z<{@!=8)p&O82iObwyYlys2hV`5pnT5TUo9e1QW0%OEP zuowyp0QNW`se(;-ot49|Tbbh~Ru?>+H&aWiCZBZE5h#nhScl@1qyE(`fzb{8tCe8q z<^aMh1O$3<_Wh?Q%Py7rP}k;cGY#A0)T8`WBy)v{$u6E)jUpqwzRciXgrpoY|6Vxj z#G6JE6W4$aMFYaJTa0QO=_ICnhlB12AnMe9#XaYJF=3E$;ryz=QI2elDS5in@_Wsz zQxszBwog!Z@A&Jn5Dt$Z!wjn$0OE(e%1VsiobR7$hY zZAK3+vl*z#k&QOkph#FJb^mbZn03)?b^6_@oHWji+dQ=uR#6*Go%XL5ldu_!Ne<_5 zvqq>6J;d5PXRPDvL5BWqARfBhny-R(&1R!lDdjp52c!xNXi~!$Wh-iBQL@7;MVYlO z*Yh}CF*~1YgAx{RdLJzpOPuSMMY{wTb+m7%ZnyE)j1RG#>B)bO_WCU*V5@Kox^)e4oY9s3ZtrP@TP2cAh|@&x17|GL z%GGwxvB=ZhjQ=`t@R6C$i8E#*Cbsz-<*|Vo@T_AZ3`xcm6`{3t&EFeZx+2 zM~n5Aj$8(o5is*~CvK@E&-KuV5_~ z9!YTz0uwkIYVLHtQjtq9!uhe$`%jyltku>}iOTu)=k{f6-!1os&*iOW;$m;f;}?sMoJC1T zuhFcG`QiP~Gu`XWItanJR+%SFhLqULwVv5;A-Ps@%uyA{FDj7%YX{=Oc&imrd!$)) zE=Ek;(ud3ttd7H0IOYijwUidCOj~C*+T?oz64HP5^3bD*Yv_>8`lCo+h^@oHRoFr4 zqNx&n*@9q#Xz!n2{WLAfd43^73c_9vv2uK$kgz}EV4A6A_5c z1=f1$mM_AqQyG}ul{1s8T&rsCi#ih;q?^u&+(5n-G%zGBzQs^tsMQs=sE6_Zk2FZ> z4%F@fVh6-s_wzJ+g+1>3!|ssXcJ+BP#f+iVQ8o6hh_B_{ZZG#MSDW6nU${=-PB&@# zp14qdfF7O0;Zp;4^5p6aa1^0ew4WMmVx66q&w;CJaN9eLbyHBA-`|_R;bFg zLiok< zV|fEv2(;}k5TD)o#oJ5)i02Y^lXOp6ghr9IX-ND0K2qP?fLeQ)h@FSCRX3TLcL#WZ zgy02N-X&is&YVa~!mL*J@NaI3XL@@~5B08Cw%Ri?6Euu~-PZ-c>#{O_>>=TUVG58I zW09w62h@;8ru*1K_CA&Qi#c)SlLkso&w#f2m5bXsY~$5KDQUC*1EEgTxl~Q5)O|Zif&!fbFJ(F~-?yueyRzmrFVG z(0s@QetWEj^j>z+F80yAHlrl+{!v@DyWu#m!n*LM)MuB+TY~XIf(_nJ1y}3Ne1hUJ zlt9jzdvvX`iL`_4RFBi;4TVZGBSnKKee2Y`hUIsIz8C-bKYzfkA2|0T_dxg502Lo= zsLHGZ*)2_E)@e`L%;R_$DkEAM$&}b@zn>6#$vgetwEIf$At<&9%qK{idulxq%~#_# z{T@!>JN@fZqZOdhr;q#us(banUXks;X+;$t?^#1 zal5@fzc}pGkp?kK*Xk6&tsbmXZgcc~vEbp~{%NOg>u#J#UzN|dVW91G^zg_}J#-~p zPCL-^5=HEo0E#iqj}2Esq}bcruacEaldE)=`m^3J z*6;Z3zzJS=^>Fd&*wp4-FLQyxD*|Z03!H^K`irk%+KO=jfP=?5t%yT z^#-n%Vn2x-@nDy^oW-zT|D_Cm+bA=KQ4-Y@Vya8yXy=Us~8v_2?xIAv;X&$^u zeZA(CPwBROrFNoy!_|HxkGJd{{`&p_H;+G|U62C#O_8>>D?mz9f{pYcC-#J5lec;r)(rx{(TH5~+6)*I#oigjRnf-Q~JUhQ#KYZ@d zZ+LtDmCcRl`#S}Z@tZVfeglL3ovLNczl!Sw{%*vK-=b4;N_z0BOQeGK?*jb{!i`Lk z+lPzK#HKg@x?aai=E3g)`%7>{#kT&8mX6=!;%7wuExw?7{oDFKPO!g&=SPF{cj^kD z@PEt61K`(n0(u^pi-Ri{`y6W5iJ$mY-Lb0pl=uGt%6|@#^D{yo0GTEzXxJyxp&<;H zdNSaxHh}JUhF}z*YnD0*jN?ZhZj>DEf9IO|6CD41_P+cgYHfu;)DB%wz5R*kW^u{X zG`gV)CDMv-P5vwd{*0Nx;BQL% zZ&>)fu>Bhr{)UBZ`~DXcq?@z)=S8r*R$$os7~j6~3Qyoi`8jzjrNTb5S?74o-?Y^D z^;d`1La_@gQaY+z1XX!XAL7ku$f(EhKbG1|>$_}(2LFq@lMiJT_)jpogue}IxZz=&%82Ecj{af>-RJp=s zILq9AF-bsdpN>+c6+#>8x*R5>ZiACG;}ioTlw_*_8CDDsn$Liz%ru~8+9^2eqkm|% zPJ2g0@TUnkRVo^09D|}&>*(uHNZ+TVSH0esVYWaipSV8%mClG=qEzoo!DGQumFd!L|V-|HCAd)W&Uy0rTtu)~~| zp5A!#Fs#@xR$)F_-C+_8cB=Ujv*WmlI=)l7mI4Qv`cttbCDflwA9Bi;dD ztR%1njFwK)g}m6y+CxPYd z?aGl!UB*j-)o0-yKGQj<8h{NV;1mS)m}#*Sa++}x@#;@CRw4ps``9{mJ`+$Cw%O)> zs}iE#r^M0XHiX5fyMFy8i>0)+a;732!2)}ts|D z9&Q-qY#wNw%Cr#hV&;MEFfaP5cLO(2XDE^XdSXGWSf$ZS z%xkZ%T*2P&l&b*v-~?K=2*u@{uxP5jIHaKy