method.py

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import json\n",
    "import nltk\n",
    "import pandas as pd\n",
    "import re\n",
    "import matplotlib.pyplot as plt \n",
    "import gensim\n",
    "import community\n",
    "import pyLDAvis.gensim\n",
    "pyLDAvis.enable_notebook()\n",
    "\n",
    "from IPython.display import display\n",
    "from gensim import corpora, models\n",
    "from string import punctuation\n",
    "from nltk.corpus import stopwords\n",
    "from datetime import datetime\n",
    "from textblob import TextBlob  \n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer as Vectorizer\n",
    "from nltk  import TweetTokenizer\n",
    "from gensim.models import CoherenceModel \n",
    "from gensim.test.utils import common_corpus, common_dictionary\n",
    "from gensim.models import HdpModel\n",
    "from networkx.algorithms.community import greedy_modularity_communities\n",
    "\n",
    "# import pyLDAvis\n",
    "# import matplotlib.pyplot as plt\n",
    "# nltk.download('stopwords')\n",
    "pd.set_option('display.max_colwidth', -1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#VARIABLES:\n",
    "english_stopwords = stopwords.words('english')\n",
    "df =[]\n",
    "#Variables para sentimentalAnalis()\n",
    "popularidad_list = []\n",
    "numeros_list = []\n",
    "numero = 0 \n",
    "\n",
    "#Variable for LDA AND HLDA\n",
    "corpus = \"\"\n",
    "elDictionary = \"\"\n",
    "ldaResult = \"\"\n",
    "modelLDA = \"\"\n",
    "list_user_mentioned = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Metod para el sentimental Análisis y dibujar la grafica\n",
    "def sentimentalAnalis(twitText):\n",
    "    global numero, popularidad_list, numeros_list\n",
    "    analisis = TextBlob(str(twitText))\n",
    "    popularidad = analisis.sentiment\n",
    "    popularidad = popularidad.polarity\n",
    "    popularidad_list.append(popularidad)\n",
    "    numeros_list.append(numero)\n",
    "    numero = numero + 1 \n",
    "    return popularidad\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "eval() arg 1 must be a string, bytes or code object",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-be04a875d0ee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0mclean_data_from_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-18-be04a875d0ee>\u001b[0m in \u001b[0;36mclean_data_from_json\u001b[0;34m(file, ntwits)\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0musr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtwt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muser_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtwit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m             \u001b[0musernames\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0musr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'screen_name'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0musr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: eval() arg 1 must be a string, bytes or code object"
     ]
    }
   ],
   "source": [
    "\n",
    "def clean_data_from_json(file,ntwits):\n",
    "    # Load the first sheet of the JSON file into a data frame\n",
    "    file = \"../FicherosJSON/P1TheDemocrats_10000.json\"\n",
    "    global df\n",
    "    usernames = []\n",
    "    sentimentOfTwritUser = []\n",
    "    df = pd.read_json(file, orient='columns')\n",
    "    df.to_csv('DataFrame.csv',encoding=\"utf-8\")\n",
    "    df = pd.read_csv('DataFrame.csv', encoding='utf-8')\n",
    "    df = df[:ntwits]\n",
    "    user_id = df['user'].tolist()\n",
    "    twit = df['text'].tolist()\n",
    "    for usr, twt in zip(user_id, twit):\n",
    "        try: \n",
    "            usernames.append(eval(str(usr))['screen_name'])\n",
    "        except ValueError:\n",
    "            print(usr)\n",
    "        sentimentOfTwritUser.append(sentimentalAnalis(twt))\n",
    "    df=df.assign(screen_name=usernames)\n",
    "    df=df.assign(sentiment=sentimentOfTwritUser)\n",
    "    \n",
    "    data = df['text'].tolist()\n",
    "\n",
    "    return data\n",
    "clean_data_from_json(\"\",5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GraficarDatos(numeros_list,popularidad_list,numero):\n",
    "    axes = plt.gca()\n",
    "    axes.set_ylim([-1, 2]) \n",
    "    \n",
    "    plt.scatter(numeros_list, popularidad_list)\n",
    "\n",
    "    popularidadPromedio = (sum(popularidad_list))/(len(popularidad_list))\n",
    "    popularidadPromedio = \"{0:.0f}%\".format(popularidadPromedio * 100)\n",
    "    time  = datetime.now().strftime(\"\")\n",
    "    # time  = datetime.now().strftime(\"A : %H:%M\\n El: %m-%d-%y\")\n",
    "    plt.text(0, 1.25, \n",
    "             \"Sentimiento promedio:  \" + str(popularidadPromedio) + \"\\n\" + time, \n",
    "             fontsize=12, \n",
    "             bbox = dict(facecolor='none', \n",
    "                         edgecolor='black', \n",
    "                         boxstyle='square, pad = 1'))\n",
    "    \n",
    "    plt.title(\"Sentimientos sobre P1THEDEMOCRATS_2000 en twitter\")\n",
    "    plt.xlabel(\"Numero de tweets\")\n",
    "    plt.ylabel(\"Sentimiento\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relaciones_network():\n",
    "    global df\n",
    "    graph=nx.DiGraph()\n",
    "    userName = df['screen_name'].tolist()\n",
    "    snetiment = df['screen_name'].tolist()\n",
    "    for usr, snt in zip(userName, snetiment):\n",
    "        graph.add_node(usr, snt=0)\n",
    "    \n",
    "    content_entities = df['entities'].tolist()\n",
    "    user_id = df['user'].tolist()\n",
    "    twit = df['text'].tolist()\n",
    "    for names, author,twit in zip(content_entities, user_id,twit):\n",
    "        sent = sentimentalAnalis(twit)\n",
    "        names = eval(names)['user_mentions']\n",
    "        author = eval(str(author))['screen_name']\n",
    "        for val in names:\n",
    "            try:\n",
    "                graph[author][val['screen_name']][\"weight\"]+=1\n",
    "                graph[author][val['screen_name']][\"sentiment\"]+= sent\n",
    "                \n",
    "            except:\n",
    "                graph.add_edge(author,val['screen_name'],weight=1,sentiment=sent)\n",
    "     \n",
    "    for var1,var2,var3 in graph.edges(data=True):\n",
    "#        print(\"Sentimiento\")\n",
    "#        print(var3.get('sentiment'))\n",
    "#        print(\"Peso\")\n",
    "#        print(var3.get('weight'))\n",
    "        result = float(var3.get('sentiment')) / float(var3.get('weight'))\n",
    "#        print(\"AAAAAAAAAAAAAAAAAAAAAA\")\n",
    "#        print(result)\n",
    "        graph[var1][var2][\"sentiment\"]=result\n",
    "    return graph\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateSentimentNode(graph):\n",
    "    atributeNode = []\n",
    "    atribute = []\n",
    "    userS = 0\n",
    "    outDegreeNode = []\n",
    "    for node in graph.nodes():\n",
    "        for outEdges in graph.out_edges(nbunch=node,data=True):\n",
    "            userS += outEdges[2]['sentiment']\n",
    "        atributeNode.append(userS)\n",
    "    for var1, var2 in graph.out_degree():\n",
    "        outDegreeNode.append(var2)\n",
    "    for var1, var2 in zip(atributeNode, outDegreeNode):\n",
    "        if var2 > 0:\n",
    "            print(\"RESULTADO\")\n",
    "            print(var1/var2)\n",
    "            atribute.append(var1/var2)\n",
    "        else:\n",
    "            atribute.append(0)\n",
    "    \n",
    "    nx.set_node_attributes(graph,atribute, 'snt')\n",
    "    return atribute\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweet_clean(tweet):\n",
    "    print('Original tweet:', tweet, '\\n')\n",
    "    tweet = str(tweet)\n",
    "    # Remove HTML special entities (e.g. &amp;)\n",
    "    tweet_no_special_entities = re.sub(r'\\&\\w*;', '', tweet)\n",
    "    tweet_no_special_entities = re.sub(r'\\@\\w*;', '', tweet)\n",
    "    #print('No special entitites:', tweet_no_special_entities, '\\n')\n",
    "    # Remove tickers\n",
    "    tweet_no_tickers = re.sub(r'\\$\\w*', '', tweet_no_special_entities)\n",
    "    #print('No tickers:', tweet_no_tickers, '\\n')\n",
    "    # Remove hyperlinks\n",
    "    tweet_no_hyperlinks = re.sub(r'https?:\\/\\/.*\\/\\w*', '', tweet_no_tickers)\n",
    "    #print('No hyperlinks:', tweet_no_hyperlinks, '\\n')\n",
    "    # Remove hashtags\n",
    "    tweet_no_hashtags = re.sub(r'#\\w*', '', tweet_no_hyperlinks)\n",
    "    #print('No hashtags:', tweet_no_hashtags, '\\n')\n",
    "    # Remove Punctuation and split 's, 't, 've with a space for filter\n",
    "    tweet_no_punctuation = re.sub(r'[' + punctuation.replace('@', '') + ']+', ' ', tweet_no_hashtags)\n",
    "    #print('No punctuation:', tweet_no_punctuation, '\\n')\n",
    "    # Remove https\n",
    "    tweet_no_https = re.sub(r'https', '', tweet_no_punctuation)\n",
    "    tweet_no_https = re.sub(r'http', '', tweet_no_punctuation)\n",
    "    #print('No https:', tweet_no_https, '\\n')\n",
    "    # Remove words with 2 or fewer letters\n",
    "    tweet_no_small_words = re.sub(r'\\b\\w{1,2}\\b', '', tweet_no_https)\n",
    "    #print('No small words:', tweet_no_small_words, '\\n')\n",
    "    # Remove whitespace (including new line characters)\n",
    "    tweet_no_whitespace = re.sub(r'\\s\\s+', ' ', tweet_no_small_words) \n",
    "    tweet_no_whitespace = tweet_no_whitespace.lstrip(' ') # Remove single space remaining at the front of the tweet.\n",
    "    #print('No whitespace:', tweet_no_whitespace, '\\n')\n",
    "\t# Remove characters beyond Basic Multilingual Plane (BMP) of Unicode:\n",
    "    # tweet_no_emojis = ''.join(c for c in tweet_no_whitespace if c <= '\\uFFFF') # Apart from emojis (plane 1), this also removes historic scripts and mathematical alphanumerics (also plane 1), ideographs (plane 2) and more.\n",
    "    # #print('No emojis:', tweet_no_whitespace, '\\n')\n",
    "    # Tokenize: Change to lowercase, reduce length and remove handles\n",
    "    tknzr = TweetTokenizer(preserve_case=False, reduce_len=True, strip_handles=True) # reduce_len changes, for example, waaaaaayyyy to waaayyy.\n",
    "    tw_list = tknzr.tokenize(tweet_no_whitespace)\n",
    "    #print('Tweet tokenize:', tw_list, '\\n')\n",
    "    # Remove stopwords\n",
    "    list_no_stopwords = [i for i in tw_list if i not in english_stopwords]\n",
    "    #print('No stop words:', list_no_stopwords, '\\n')\n",
    "    # \n",
    "\n",
    "    # Final filtered tweet\n",
    "    tweet_filtered =' '.join(list_no_stopwords)\n",
    "\n",
    "    print ('Final tweet: ', tweet_filtered)\n",
    "    \n",
    "    return(tweet_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ldaMethod(data,topics ):\n",
    "    global corpus, elDictionary, ldaResult, modelLDA\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "\n",
    "    # Create p_stemmer of class PorterStemmer\n",
    "    p_stemmer = PorterStemmer()\n",
    "\n",
    "    # list for tokenized twiits in loop\n",
    "    palabras = []\n",
    "    for i in data:\n",
    "        raw = i.lower()\n",
    "        tokens = tokenizer.tokenize(raw)\n",
    "        # stem tokens\n",
    "        stemmed_tokens = [p_stemmer.stem(word) for word in tokens]\n",
    "        \n",
    "        # add tokens to list\n",
    "        palabras.append(stemmed_tokens)\n",
    "        # twit tokenizado en documento \n",
    "    elDictionary = corpora.Dictionary(palabras)\n",
    "\n",
    "    #token en document-termino matriz\n",
    "    corpus = [elDictionary.doc2bow(text) for text in palabras]\n",
    "    \n",
    "    # Generacion LDA\n",
    "    ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=topics, id2word = elDictionary, passes=20)\n",
    "    modelLDA = ldamodel\n",
    "    print (\"LDA\")\n",
    "    ldaResult = ldamodel.print_topics(5)\n",
    "    print (ldaResult)\n",
    "\n",
    "    print (\"PERPLEXITY\")\n",
    "    print (ldamodel.log_perplexity(corpus))\n",
    "\n",
    "    coherence_model_lda = CoherenceModel(model=ldamodel, texts=palabras, dictionary=elDictionary, coherence='c_v')\n",
    "    coherence_lda = coherence_model_lda.get_coherence()\n",
    "\n",
    "    print (\"COHERENCE:\" )\n",
    "    print (coherence_lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hlda(corpus, eldictionary, topic, probably_words):\n",
    "    hdp = HdpModel(corpus, eldictionary)\n",
    "    topic_info = hdp.print_topics(num_topics=20, num_words=10)\n",
    "    print (topic_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def graficModeLDA():\n",
    "    probabilidad_del_topic = []\n",
    "    print (ldaResult)\n",
    "    for key,val in ldaResult:\n",
    "        val = re.findall(r'\\d*\\.\\d*',val)\n",
    "        val = [i.encode('ascii', 'ignore') for i in val]\n",
    "        a = 0\n",
    "        for i in val:\n",
    "            a+=float(i)\n",
    "\n",
    "        probabilidad_del_topic.append((key,a))\n",
    "\n",
    "    print (probabilidad_del_topic)\n",
    "    d = elDictionary\n",
    "    c = corpus\n",
    "    lda = modelLDA\n",
    "    data = pyLDAvis.gensim.prepare(lda, c, d)\n",
    "    data\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}