Generalize.py

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn \n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "import matplotlib.pyplot as plt\n",
    "import tqdm\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import accuracy_score\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "import regex as re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper functions\n",
    "def clean_text(text, remove_nums=True):\n",
    "    if remove_nums:\n",
    "        # Remove all numbers \n",
    "        result = re.sub(r'[0-9\\.]+', '', text)\n",
    "        # Lowercase and remove non-alphanumeric characters\n",
    "        return ''.join([char.lower() for char in result if char.isalnum() or char.isspace()])\n",
    "    else:\n",
    "        return ''.join([char.lower() for char in text if char.isalnum() or char.isspace()])\n",
    "\n",
    "def lemmatize_text(text, lemmatizer):\n",
    "    return ' '.join([lemmatizer.lemmatize(word) for word in text.split()])\n",
    "\n",
    "def preprocessing_texts(df, column_name, apply_lemmatization, lemmatizer):\n",
    "    texts = df[column_name].apply(clean_text)\n",
    "    if apply_lemmatization:\n",
    "        texts = texts.apply(lambda x: lemmatize_text(x, lemmatizer))\n",
    "    return texts\n",
    "\n",
    "def vectorize_datasets(train_df, test_df, vectorizer_type='count', ngram_range=(1, 1), remove_stopwords=False, apply_lemmatization=False):\n",
    "    lemmatizer = WordNetLemmatizer() if apply_lemmatization else None\n",
    "    stop_words = 'english' if remove_stopwords else None\n",
    "\n",
    "    # Choose Vectorizer: 'count' or 'tfidf'\n",
    "    if vectorizer_type == 'count':\n",
    "        vectorizer = CountVectorizer(stop_words=stop_words, ngram_range=ngram_range)\n",
    "    elif vectorizer_type == 'tfidf':\n",
    "        vectorizer = TfidfVectorizer(stop_words=stop_words, ngram_range=ngram_range)\n",
    "    else:\n",
    "        raise ValueError(\"Invalid vectorizer type\")\n",
    "\n",
    "    # Clean and lemmatize text\n",
    "    train_texts = preprocessing_texts(train_df, column_name='content', apply_lemmatization=apply_lemmatization, lemmatizer=lemmatizer)\n",
    "    test_texts = preprocessing_texts(test_df, column_name='content', apply_lemmatization=apply_lemmatization, lemmatizer=lemmatizer)\n",
    "\n",
    "    # Vectorize the text data\n",
    "    train_vectors = vectorizer.fit_transform(train_texts)\n",
    "    test_vectors = vectorizer.transform(test_texts)\n",
    "\n",
    "    # Extract the label column (real=1, fake=0)\n",
    "    y_train = train_df['label']\n",
    "    y_test = test_df['label']\n",
    "\n",
    "    return train_vectors, test_vectors, y_train, y_test, vectorizer\n",
    "\n",
    "def bleaching(textdf):\n",
    "    # Uses the clean_text() function to completely sanitize the text inside a pandas dataframe \n",
    "    for idx in textdf.index:\n",
    "        cleaned_text = clean_text(textdf.loc[idx, 'content'], remove_nums=True)\n",
    "        textdf.loc[idx, 'content'] = cleaned_text\n",
    "    return textdf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the data\n",
    "factsSource = './testfacts.txt'\n",
    "fakesSource = './testfakes.txt'\n",
    "\n",
    "with open(factsSource, 'r') as f:\n",
    "    facts = f.readlines()\n",
    "\n",
    "with open(fakesSource, 'r') as f:\n",
    "    fakes = f.readlines()\n",
    "\n",
    "data = pd.DataFrame({\n",
    "    'content': facts + fakes,\n",
    "    'label': [1] * len(facts) + [0] * len(fakes)  # Label facts as 1, fakes as 0\n",
    "})\n",
    "\n",
    "# Shuffle data\n",
    "data = data.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "# Split data\n",
    "traindf, testdf = train_test_split(data, test_size=0.2, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vectorizing the data\n",
    "Xtrain, Xtest, ytrain, ytest, vectorizer = vectorize_datasets(\n",
    "    train_df=traindf,\n",
    "    test_df=testdf,\n",
    "    vectorizer_type='tfidf',  # You can switch between 'count' and 'tfidf'\n",
    "    ngram_range=(1, 2),  # Use unigrams and bigrams\n",
    "    remove_stopwords=False,  # Remove stop words\n",
    "    apply_lemmatization=False  # Apply lemmatization\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes Accuracy: 0.7629\n",
      "SVM Accuracy: 0.8557\n",
      "Logistic Regression Accuracy: 0.8514\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "# Naive Bayes, SVM, and Logistic Regression using sklearn\n",
    "def sklearn_classifiers(Xtrain, ytrain):\n",
    "    classifiers = {\n",
    "        \"Naive Bayes\": MultinomialNB(),\n",
    "        \"SVM\": SVC(),\n",
    "        \"Logistic Regression\": LogisticRegression(max_iter=1000)\n",
    "    }\n",
    "    \n",
    "    for name, clf in classifiers.items():\n",
    "        scores = cross_val_score(clf, Xtrain, ytrain, cv=5)\n",
    "        print(f\"{name} Accuracy: {np.mean(scores):.4f}\")\n",
    "\n",
    "# Running the classifiers\n",
    "sklearn_classifiers(Xtrain, ytrain)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running GridSearchCV for NaiveBayes\n",
      "Best Score for NaiveBayes: 0.8071\n",
      "Best Parameters for NaiveBayes: {'alpha': 0.1}\n",
      "\n",
      "Running GridSearchCV for SVM\n",
      "Best Score for SVM: 0.8700\n",
      "Best Parameters for SVM: {'C': 1.0, 'gamma': 'scale', 'kernel': 'linear'}\n",
      "\n",
      "Running GridSearchCV for Logistic\n",
      "Best Score for Logistic: 0.8514\n",
      "Best Parameters for Logistic: {'C': 0.9526315789473684, 'max_iter': 100, 'solver': 'liblinear'}\n",
      "\n",
      "Evaluating NaiveBayes with best hyperparameters\n",
      "Test Accuracy for NaiveBayes: 0.8267\n",
      "\n",
      "Evaluating SVM with best hyperparameters\n",
      "Test Accuracy for SVM: 0.8400\n",
      "\n",
      "Evaluating Logistic with best hyperparameters\n",
      "Test Accuracy for Logistic: 0.8233\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Hyperparameter tuning\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "def hyperparameter_tuning(Xtrain, ytrain, classifiers):\n",
    "\n",
    "    # Hyperparameter ranges to test \n",
    "    alpha = np.linspace(0.1, 1, 20)\n",
    "    C = np.linspace(0.1, 1, 20)   \n",
    "    \n",
    "    # Define hyperparameter grids for each classifier\n",
    "    param_grids = {\n",
    "    \"NaiveBayes\": {\n",
    "        'alpha': alpha  # Laplace smoothing parameter\n",
    "    },\n",
    "    \"SVM\": {\n",
    "        'C': C,  # Regularization \n",
    "        'kernel': ['linear', 'rbf'],  \n",
    "        'gamma': ['scale', 'auto']  # Kernel coefficient\n",
    "    },\n",
    "    \"Logistic\": {\n",
    "        'C': C,  # Regularization\n",
    "        'solver': ['liblinear', 'lbfgs'],  # Optimization\n",
    "        'max_iter': [100, 200, 300]  \n",
    "    }\n",
    "    }\n",
    "\n",
    "\n",
    "    # Test hyperparameter \n",
    "    best_params = {}\n",
    "    for name, clf in classifiers.items():\n",
    "        print(f\"Running GridSearchCV for {name}\")\n",
    "        grid_search = GridSearchCV(clf, param_grids[name], cv=5, scoring='accuracy')\n",
    "        grid_search.fit(Xtrain, ytrain)\n",
    "\n",
    "        # Store the best parameters and scores\n",
    "        best_params[name] = {\n",
    "            'best_score': grid_search.best_score_,\n",
    "            'best_params': grid_search.best_params_\n",
    "        }\n",
    "        print(f\"Best Score for {name}: {grid_search.best_score_:.4f}\")\n",
    "        print(f\"Best Parameters for {name}: {grid_search.best_params_}\\n\")\n",
    "        \n",
    "\n",
    "    return best_params\n",
    "\n",
    "\n",
    "# Retrain models with the best hyperparameters and evaluate on the test set\n",
    "def evaluate_best_models(Xtrain, Xtest, ytrain, ytest, best_params):\n",
    "    for name, clf in classifiers.items():\n",
    "        print(f\"Evaluating {name} with best hyperparameters\")\n",
    "        \n",
    "        # Set the best params\n",
    "        clf.set_params(**best_params[name]['best_params'])\n",
    "        clf.fit(Xtrain, ytrain)\n",
    "        \n",
    "        # Evaluate the model on the test set\n",
    "        test_accuracy = clf.score(Xtest, ytest)\n",
    "        print(f\"Test Accuracy for {name}: {test_accuracy:.4f}\\n\")\n",
    "\n",
    "\n",
    "\n",
    "# Hyperparameter tuning\n",
    "classifiers = {\n",
    "     \"NaiveBayes\": MultinomialNB(),\n",
    "     \"SVM\": SVC(),\n",
    "     \"Logistic\": LogisticRegression()\n",
    " }\n",
    "\n",
    "# Find the best params\n",
    "best_params = hyperparameter_tuning(Xtrain, ytrain, classifiers)\n",
    "# Retrain and eval the models with the best params \n",
    "evaluate_best_models(Xtrain, Xtest, ytrain, ytest, best_params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common real fact words: [('is', 364), ('the', 331), ('of', 231), ('in', 197), ('and', 133), ('city', 98), ('one', 81), ('world', 68), ('to', 67), ('has', 62)]\n",
      "Most common fake fact words: [('the', 395), ('is', 372), ('of', 194), ('world', 166), ('in', 153), ('city', 118), ('largest', 83), ('to', 66), ('has', 58), ('was', 58)]\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "# Vectorize without stopword removal or lemmatization to inspect tokens\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(data['content'])  # X is a sparse matrix\n",
    "\n",
    "# Get feature (word) names\n",
    "feature_names = vectorizer.get_feature_names()\n",
    "\n",
    "# Separate the real and fake labels\n",
    "real_indices = data[data['label'] == 1].index\n",
    "fake_indices = data[data['label'] == 0].index\n",
    "\n",
    "# Convert sparse matrix rows to actual tokens\n",
    "def get_tokens_from_csr(csr_matrix_row):\n",
    "    # Find the indices of the non-zero elements in the sparse matrix row\n",
    "    tokens_indices = csr_matrix_row.nonzero()[1]\n",
    "    return [feature_names[i] for i in tokens_indices]\n",
    "\n",
    "# Get the words for real and fake facts by extracting non-zero tokens\n",
    "real_words = [word for idx in real_indices for word in get_tokens_from_csr(X[idx])]\n",
    "fake_words = [word for idx in fake_indices for word in get_tokens_from_csr(X[idx])]\n",
    "\n",
    "# Count the most common tokens\n",
    "real_word_count = Counter(real_words)\n",
    "fake_word_count = Counter(fake_words)\n",
    "\n",
    "# Display the most common words\n",
    "print(\"Most common real fact words:\", real_word_count.most_common(10))\n",
    "print(\"Most common fake fact words:\", fake_word_count.most_common(10))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}