From af67c6aabd85e98cce86be8dfc5f448fb454fd2f Mon Sep 17 00:00:00 2001 From: Basel Mousi Date: Mon, 21 Aug 2023 11:52:48 +0300 Subject: [PATCH] Added a notebook for text excerpts extraction --- Activation-Extraction.ipynb | 650 ++++++++++++++++++++++++++++++++++++ 1 file changed, 650 insertions(+) create mode 100644 Activation-Extraction.ipynb diff --git a/Activation-Extraction.ipynb b/Activation-Extraction.ipynb new file mode 100644 index 0000000..c03a912 --- /dev/null +++ b/Activation-Extraction.ipynb @@ -0,0 +1,650 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2b80f86e", + "metadata": {}, + "source": [ + "# Extracting Top Activating Text Excerpts for Neurons " + ] + }, + { + "cell_type": "markdown", + "id": "480e12b9", + "metadata": {}, + "source": [ + "This notebook serves as a guide to extracting the most highly activating text excerpts for a specified neuron. We will leverage [NeuroX](https://neurox.qcri.org/) which is a framework that aims to interpret deep NLP models and increase the transparency of their inner workings and predictions. Through the utilization of such code, one can get the top activating text excerpts for a given neuron. Consequently, this capability facilitates the creation of a Neuron Record, enabling the generation of explanations tailored to the aforementioned neuron." + ] + }, + { + "cell_type": "markdown", + "id": "3996f026", + "metadata": {}, + "source": [ + "# Import Libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9822271", + "metadata": {}, + "outputs": [], + "source": [ + "import neurox.data.extraction.transformers_extractor as transformers_extractor\n", + "from neurox.data import loader as loader\n", + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "id": "98ae2ffc", + "metadata": {}, + "source": [ + "# Inspect Toy Data" + ] + }, + { + "cell_type": "markdown", + "id": "0da55e57", + "metadata": {}, + "source": [ + "The toy data file contains random sentences generated using ChatGPT" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6069674", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The sun rises in the east.\r\n", + "Birds fly across the sky.\r\n", + "The river flows calmly.\r\n", + "Children play in the park.\r\n", + "Books are knowledge.\r\n", + "The wind blows.\r\n", + "The cat meows.\r\n", + "Cars drive on the road.\r\n", + "The moon shines brightly.\r\n", + "The ocean waves crash.\r\n" + ] + } + ], + "source": [ + "!head -n 10 toy-data.txt" + ] + }, + { + "cell_type": "markdown", + "id": "a1d0b431", + "metadata": {}, + "source": [ + "# Define Variables For Activation Extraction" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b6a3161", + "metadata": {}, + "outputs": [], + "source": [ + "data_path = \"toy-data.txt\"\n", + "activations_path = \"activations.json\"\n", + "model = \"bert-base-cased\"\n", + "max_sent_l=512" + ] + }, + { + "cell_type": "markdown", + "id": "c1820fc3", + "metadata": {}, + "source": [ + "# Extract Activations " + ] + }, + { + "cell_type": "markdown", + "id": "7b876350", + "metadata": {}, + "source": [ + "We will be extracting the activations using the [NeuroX transformers Extractor](https://neurox.qcri.org/docs/neurox.data.extraction.html)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2d872b1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading model: bert-base-cased\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading input corpus\n", + "Preparing output file\n", + "Extracting representations from model\n", + "Sentence : \"The sun rises in the east.\"\n", + "Original (006): ['The', 'sun', 'rises', 'in', 'the', 'east.']\n", + "Tokenized (009): ['[CLS]', 'The', 'sun', 'rises', 'in', 'the', 'east', '.', '[SEP]']\n", + "Filtered (007): ['The', 'sun', 'rises', 'in', 'the', 'east', '.']\n", + "Detokenized (006): ['The', 'sun', 'rises', 'in', 'the', 'east.']\n", + "Counter: 7\n", + "===================================================================\n", + "Hidden states: (13, 6, 768)\n", + "# Extracted words: 6\n", + "Sentence : \"Birds fly across the sky.\"\n", + "Original (005): ['Birds', 'fly', 'across', 'the', 'sky.']\n", + "Tokenized (008): ['[CLS]', 'Birds', 'fly', 'across', 'the', 'sky', '.', '[SEP]']\n", + "Filtered (006): ['Birds', 'fly', 'across', 'the', 'sky', '.']\n", + "Detokenized (005): ['Birds', 'fly', 'across', 'the', 'sky.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"The river flows calmly.\"\n", + "Original (004): ['The', 'river', 'flows', 'calmly.']\n", + "Tokenized (007): ['[CLS]', 'The', 'river', 'flows', 'calmly', '.', '[SEP]']\n", + "Filtered (005): ['The', 'river', 'flows', 'calmly', '.']\n", + "Detokenized (004): ['The', 'river', 'flows', 'calmly.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"Children play in the park.\"\n", + "Original (005): ['Children', 'play', 'in', 'the', 'park.']\n", + "Tokenized (008): ['[CLS]', 'Children', 'play', 'in', 'the', 'park', '.', '[SEP]']\n", + "Filtered (006): ['Children', 'play', 'in', 'the', 'park', '.']\n", + "Detokenized (005): ['Children', 'play', 'in', 'the', 'park.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"Books are knowledge.\"\n", + "Original (003): ['Books', 'are', 'knowledge.']\n", + "Tokenized (006): ['[CLS]', 'Books', 'are', 'knowledge', '.', '[SEP]']\n", + "Filtered (004): ['Books', 'are', 'knowledge', '.']\n", + "Detokenized (003): ['Books', 'are', 'knowledge.']\n", + "Counter: 4\n", + "===================================================================\n", + "Hidden states: (13, 3, 768)\n", + "# Extracted words: 3\n", + "Sentence : \"The wind blows.\"\n", + "Original (003): ['The', 'wind', 'blows.']\n", + "Tokenized (006): ['[CLS]', 'The', 'wind', 'blows', '.', '[SEP]']\n", + "Filtered (004): ['The', 'wind', 'blows', '.']\n", + "Detokenized (003): ['The', 'wind', 'blows.']\n", + "Counter: 4\n", + "===================================================================\n", + "Hidden states: (13, 3, 768)\n", + "# Extracted words: 3\n", + "Sentence : \"The cat meows.\"\n", + "Original (003): ['The', 'cat', 'meows.']\n", + "Tokenized (007): ['[CLS]', 'The', 'cat', 'me', '##ows', '.', '[SEP]']\n", + "Filtered (005): ['The', 'cat', 'me', '##ows', '.']\n", + "Detokenized (003): ['The', 'cat', 'me##ows.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 3, 768)\n", + "# Extracted words: 3\n", + "Sentence : \"Cars drive on the road.\"\n", + "Original (005): ['Cars', 'drive', 'on', 'the', 'road.']\n", + "Tokenized (008): ['[CLS]', 'Cars', 'drive', 'on', 'the', 'road', '.', '[SEP]']\n", + "Filtered (006): ['Cars', 'drive', 'on', 'the', 'road', '.']\n", + "Detokenized (005): ['Cars', 'drive', 'on', 'the', 'road.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"The moon shines brightly.\"\n", + "Original (004): ['The', 'moon', 'shines', 'brightly.']\n", + "Tokenized (008): ['[CLS]', 'The', 'moon', 'shine', '##s', 'brightly', '.', '[SEP]']\n", + "Filtered (006): ['The', 'moon', 'shine', '##s', 'brightly', '.']\n", + "Detokenized (004): ['The', 'moon', 'shine##s', 'brightly.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The ocean waves crash.\"\n", + "Original (004): ['The', 'ocean', 'waves', 'crash.']\n", + "Tokenized (007): ['[CLS]', 'The', 'ocean', 'waves', 'crash', '.', '[SEP]']\n", + "Filtered (005): ['The', 'ocean', 'waves', 'crash', '.']\n", + "Detokenized (004): ['The', 'ocean', 'waves', 'crash.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"People walk in the street.\"\n", + "Original (005): ['People', 'walk', 'in', 'the', 'street.']\n", + "Tokenized (008): ['[CLS]', 'People', 'walk', 'in', 'the', 'street', '.', '[SEP]']\n", + "Filtered (006): ['People', 'walk', 'in', 'the', 'street', '.']\n", + "Detokenized (005): ['People', 'walk', 'in', 'the', 'street.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"Trees sway in the breeze.\"\n", + "Original (005): ['Trees', 'sway', 'in', 'the', 'breeze.']\n", + "Tokenized (008): ['[CLS]', 'Trees', 'sway', 'in', 'the', 'breeze', '.', '[SEP]']\n", + "Filtered (006): ['Trees', 'sway', 'in', 'the', 'breeze', '.']\n", + "Detokenized (005): ['Trees', 'sway', 'in', 'the', 'breeze.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"The clock ticks steadily.\"\n", + "Original (004): ['The', 'clock', 'ticks', 'steadily.']\n", + "Tokenized (008): ['[CLS]', 'The', 'clock', 't', '##icks', 'steadily', '.', '[SEP]']\n", + "Filtered (006): ['The', 'clock', 't', '##icks', 'steadily', '.']\n", + "Detokenized (004): ['The', 'clock', 't##icks', 'steadily.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"Rain falls from the clouds.\"\n", + "Original (005): ['Rain', 'falls', 'from', 'the', 'clouds.']\n", + "Tokenized (008): ['[CLS]', 'Rain', 'falls', 'from', 'the', 'clouds', '.', '[SEP]']\n", + "Filtered (006): ['Rain', 'falls', 'from', 'the', 'clouds', '.']\n", + "Detokenized (005): ['Rain', 'falls', 'from', 'the', 'clouds.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"Fish swim in the sea.\"\n", + "Original (005): ['Fish', 'swim', 'in', 'the', 'sea.']\n", + "Tokenized (008): ['[CLS]', 'Fish', 'swim', 'in', 'the', 'sea', '.', '[SEP]']\n", + "Filtered (006): ['Fish', 'swim', 'in', 'the', 'sea', '.']\n", + "Detokenized (005): ['Fish', 'swim', 'in', 'the', 'sea.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"The door creaks open.\"\n", + "Original (004): ['The', 'door', 'creaks', 'open.']\n", + "Tokenized (009): ['[CLS]', 'The', 'door', 'c', '##reak', '##s', 'open', '.', '[SEP]']\n", + "Filtered (007): ['The', 'door', 'c', '##reak', '##s', 'open', '.']\n", + "Detokenized (004): ['The', 'door', 'c##reak##s', 'open.']\n", + "Counter: 7\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The fire crackles warmly.\"\n", + "Original (004): ['The', 'fire', 'crackles', 'warmly.']\n", + "Tokenized (008): ['[CLS]', 'The', 'fire', 'crack', '##les', 'warmly', '.', '[SEP]']\n", + "Filtered (006): ['The', 'fire', 'crack', '##les', 'warmly', '.']\n", + "Detokenized (004): ['The', 'fire', 'crack##les', 'warmly.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"Birds sing in the morning.\"\n", + "Original (005): ['Birds', 'sing', 'in', 'the', 'morning.']\n", + "Tokenized (008): ['[CLS]', 'Birds', 'sing', 'in', 'the', 'morning', '.', '[SEP]']\n", + "Filtered (006): ['Birds', 'sing', 'in', 'the', 'morning', '.']\n", + "Detokenized (005): ['Birds', 'sing', 'in', 'the', 'morning.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"The train arrives on time.\"\n", + "Original (005): ['The', 'train', 'arrives', 'on', 'time.']\n", + "Tokenized (008): ['[CLS]', 'The', 'train', 'arrives', 'on', 'time', '.', '[SEP]']\n", + "Filtered (006): ['The', 'train', 'arrives', 'on', 'time', '.']\n", + "Detokenized (005): ['The', 'train', 'arrives', 'on', 'time.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 5, 768)\n", + "# Extracted words: 5\n", + "Sentence : \"Bees buzz around flowers.\"\n", + "Original (004): ['Bees', 'buzz', 'around', 'flowers.']\n", + "Tokenized (008): ['[CLS]', 'Bee', '##s', 'buzz', 'around', 'flowers', '.', '[SEP]']\n", + "Filtered (006): ['Bee', '##s', 'buzz', 'around', 'flowers', '.']\n", + "Detokenized (004): ['Bee##s', 'buzz', 'around', 'flowers.']\n", + "Counter: 6\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sentence : \"The flower is beautiful.\"\n", + "Original (004): ['The', 'flower', 'is', 'beautiful.']\n", + "Tokenized (007): ['[CLS]', 'The', 'flower', 'is', 'beautiful', '.', '[SEP]']\n", + "Filtered (005): ['The', 'flower', 'is', 'beautiful', '.']\n", + "Detokenized (004): ['The', 'flower', 'is', 'beautiful.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The sunset is stunning.\"\n", + "Original (004): ['The', 'sunset', 'is', 'stunning.']\n", + "Tokenized (007): ['[CLS]', 'The', 'sunset', 'is', 'stunning', '.', '[SEP]']\n", + "Filtered (005): ['The', 'sunset', 'is', 'stunning', '.']\n", + "Detokenized (004): ['The', 'sunset', 'is', 'stunning.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The mountain is pretty.\"\n", + "Original (004): ['The', 'mountain', 'is', 'pretty.']\n", + "Tokenized (007): ['[CLS]', 'The', 'mountain', 'is', 'pretty', '.', '[SEP]']\n", + "Filtered (005): ['The', 'mountain', 'is', 'pretty', '.']\n", + "Detokenized (004): ['The', 'mountain', 'is', 'pretty.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The model is handsome.\"\n", + "Original (004): ['The', 'model', 'is', 'handsome.']\n", + "Tokenized (007): ['[CLS]', 'The', 'model', 'is', 'handsome', '.', '[SEP]']\n", + "Filtered (005): ['The', 'model', 'is', 'handsome', '.']\n", + "Detokenized (004): ['The', 'model', 'is', 'handsome.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The artwork is beautiful.\"\n", + "Original (004): ['The', 'artwork', 'is', 'beautiful.']\n", + "Tokenized (007): ['[CLS]', 'The', 'artwork', 'is', 'beautiful', '.', '[SEP]']\n", + "Filtered (005): ['The', 'artwork', 'is', 'beautiful', '.']\n", + "Detokenized (004): ['The', 'artwork', 'is', 'beautiful.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The beach is stunning.\"\n", + "Original (004): ['The', 'beach', 'is', 'stunning.']\n", + "Tokenized (007): ['[CLS]', 'The', 'beach', 'is', 'stunning', '.', '[SEP]']\n", + "Filtered (005): ['The', 'beach', 'is', 'stunning', '.']\n", + "Detokenized (004): ['The', 'beach', 'is', 'stunning.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The dress is pretty.\"\n", + "Original (004): ['The', 'dress', 'is', 'pretty.']\n", + "Tokenized (007): ['[CLS]', 'The', 'dress', 'is', 'pretty', '.', '[SEP]']\n", + "Filtered (005): ['The', 'dress', 'is', 'pretty', '.']\n", + "Detokenized (004): ['The', 'dress', 'is', 'pretty.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The actor is handsome.\"\n", + "Original (004): ['The', 'actor', 'is', 'handsome.']\n", + "Tokenized (007): ['[CLS]', 'The', 'actor', 'is', 'handsome', '.', '[SEP]']\n", + "Filtered (005): ['The', 'actor', 'is', 'handsome', '.']\n", + "Detokenized (004): ['The', 'actor', 'is', 'handsome.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The garden is beautiful.\"\n", + "Original (004): ['The', 'garden', 'is', 'beautiful.']\n", + "Tokenized (007): ['[CLS]', 'The', 'garden', 'is', 'beautiful', '.', '[SEP]']\n", + "Filtered (005): ['The', 'garden', 'is', 'beautiful', '.']\n", + "Detokenized (004): ['The', 'garden', 'is', 'beautiful.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The view is stunning.\"\n", + "Original (004): ['The', 'view', 'is', 'stunning.']\n", + "Tokenized (007): ['[CLS]', 'The', 'view', 'is', 'stunning', '.', '[SEP]']\n", + "Filtered (005): ['The', 'view', 'is', 'stunning', '.']\n", + "Detokenized (004): ['The', 'view', 'is', 'stunning.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The baby is cute.\"\n", + "Original (004): ['The', 'baby', 'is', 'cute.']\n", + "Tokenized (007): ['[CLS]', 'The', 'baby', 'is', 'cute', '.', '[SEP]']\n", + "Filtered (005): ['The', 'baby', 'is', 'cute', '.']\n", + "Detokenized (004): ['The', 'baby', 'is', 'cute.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The painting is beautiful.\"\n", + "Original (004): ['The', 'painting', 'is', 'beautiful.']\n", + "Tokenized (007): ['[CLS]', 'The', 'painting', 'is', 'beautiful', '.', '[SEP]']\n", + "Filtered (005): ['The', 'painting', 'is', 'beautiful', '.']\n", + "Detokenized (004): ['The', 'painting', 'is', 'beautiful.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The river is pretty.\"\n", + "Original (004): ['The', 'river', 'is', 'pretty.']\n", + "Tokenized (007): ['[CLS]', 'The', 'river', 'is', 'pretty', '.', '[SEP]']\n", + "Filtered (005): ['The', 'river', 'is', 'pretty', '.']\n", + "Detokenized (004): ['The', 'river', 'is', 'pretty.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The woman is stunning.\"\n", + "Original (004): ['The', 'woman', 'is', 'stunning.']\n", + "Tokenized (007): ['[CLS]', 'The', 'woman', 'is', 'stunning', '.', '[SEP]']\n", + "Filtered (005): ['The', 'woman', 'is', 'stunning', '.']\n", + "Detokenized (004): ['The', 'woman', 'is', 'stunning.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n", + "Sentence : \"The puppy is cute.\"\n", + "Original (004): ['The', 'puppy', 'is', 'cute.']\n", + "Tokenized (007): ['[CLS]', 'The', 'puppy', 'is', 'cute', '.', '[SEP]']\n", + "Filtered (005): ['The', 'puppy', 'is', 'cute', '.']\n", + "Detokenized (004): ['The', 'puppy', 'is', 'cute.']\n", + "Counter: 5\n", + "===================================================================\n", + "Hidden states: (13, 4, 768)\n", + "# Extracted words: 4\n" + ] + } + ], + "source": [ + "transformers_extractor.extract_representations(model, data_path, activations_path)" + ] + }, + { + "cell_type": "markdown", + "id": "1e185639", + "metadata": {}, + "source": [ + "# Load Activations" + ] + }, + { + "cell_type": "markdown", + "id": "b7978564", + "metadata": {}, + "source": [ + "Load the activations using the [NeuroX Data Loader](https://neurox.readthedocs.io/en/latest/neurox.data.html#module-neurox.data.loader)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2fd9edd9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading json activations from activations.json...\n", + "35 13.0\n" + ] + } + ], + "source": [ + "# Load Activations and tokens\n", + "activations, num_layers = loader.load_activations(activations_path)\n", + "tokens = loader.load_data(data_path, data_path, activations, max_sent_l=max_sent_l)\n", + "num_neurons_per_layer = int(activations[0].shape[1] / num_layers)" + ] + }, + { + "cell_type": "markdown", + "id": "e52b5d3c", + "metadata": {}, + "source": [ + "# Define Variables For Getting The Top Text Excerpts For A Specific Neuron" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21567551", + "metadata": {}, + "outputs": [], + "source": [ + "neuron = 133\n", + "layer = 4\n", + "n = 5 # consider top 5 text excerpts for the neuron" + ] + }, + { + "cell_type": "markdown", + "id": "d075b6c1", + "metadata": {}, + "source": [ + "# Get Max Activation Records" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "795f7b5a", + "metadata": {}, + "outputs": [], + "source": [ + "max_activations_record = {}\n", + "max_tokens_record = {}\n", + "for sent_idx, sent in enumerate(tokens[\"source\"]):\n", + " max_activations_record[sent_idx] = np.max(activations[sent_idx][:,num_neurons_per_layer*layer + neuron])\n", + " max_tokens_record[sent_idx] = sent[np.argmax(activations[sent_idx][:,num_neurons_per_layer*layer + neuron])]" + ] + }, + { + "cell_type": "markdown", + "id": "0eeb923a", + "metadata": {}, + "source": [ + "# Sort the Activations " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a20c0c0f", + "metadata": {}, + "outputs": [], + "source": [ + "# sort activations obtained and choose the top n activating records \n", + "sorted_acts = sorted(max_activations_record.items(), key = lambda x: x[1], reverse = True)[:n]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be3bb650", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[(18, 1.6492859), (7, 1.1502558), (13, 1.0802596), (10, 1.0725111), (3, 0.9881723)]\n" + ] + } + ], + "source": [ + "print(sorted_acts)" + ] + }, + { + "cell_type": "markdown", + "id": "43aa4199", + "metadata": {}, + "source": [ + "# Print top N Activating text Excerpts For The Selected Neuron" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0dfbc88", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[('The', 0.0783667266368866), ('train', -0.17363914847373962), ('arrives', -0.9608103036880493), ('on', 1.649285912513733), ('time.', 0.2141266018152237)]\n", + "[('Cars', 0.13900332152843475), ('drive', 0.2723767161369324), ('on', 1.150255799293518), ('the', 0.21985113620758057), ('road.', -0.09761186689138412)]\n", + "[('Rain', 0.5524147748947144), ('falls', 1.0802595615386963), ('from', 0.2041938751935959), ('the', 0.77774977684021), ('clouds.', 0.047311119735240936)]\n", + "[('People', -0.2982337772846222), ('walk', 0.07899212092161179), ('in', 1.072511076927185), ('the', 0.18185363709926605), ('street.', 0.0007843400235287845)]\n", + "[('Children', 0.13210095465183258), ('play', 0.27250510454177856), ('in', 0.9881722927093506), ('the', 0.6650263667106628), ('park.', 0.030416980385780334)]\n" + ] + } + ], + "source": [ + "# get top 5 excerpts corresponding to the neuron\n", + "for elem in sorted_acts: \n", + " sent_idx = elem[0]\n", + " sent = tokens[\"source\"][sent_idx]\n", + " activation_values = activations[sent_idx][:,num_neurons_per_layer*layer + neuron]\n", + " print(list(zip(sent, activation_values.tolist())))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "gpt_annotation", + "language": "python", + "name": "gpt_annotation" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}