visualizations.R

# ==================================================================================
# CTR Prediction - Data Visualization and Model Analysis
# ==================================================================================
# Author: Tracey Thanh Ho
# Purpose: Generate comprehensive visualizations for exploratory data analysis,
#          model performance evaluation, and hyperparameter tuning analysis
# ==================================================================================

# Load Required Libraries
library(ggplot2)      # Advanced plotting
library(corrplot)     # Correlation matrices
library(gridExtra)    # Multiple plot layouts
library(reshape2)     # Data reshaping for visualization
library(dplyr)        # Data manipulation
library(scales)       # Axis formatting

# Set plotting theme
theme_set(theme_minimal(base_size = 12))

# ==================================================================================
# SECTION 1: EXPLORATORY DATA ANALYSIS (EDA)
# ==================================================================================

# Load data (adjust path as needed)
# analysis_data <- read.csv("~/Downloads/predicting-clicks/analysis_data.csv")

# ----------------------------------------------------------------------------------
# 1.1 Target Variable Distribution
# ----------------------------------------------------------------------------------
plot_target_distribution <- function(data) {
  p <- ggplot(data, aes(x = CTR)) +
    geom_histogram(bins = 50, fill = "#2E86AB", alpha = 0.7, color = "white") +
    geom_density(aes(y = ..count.. * (max(..count..) / max(..density..))), 
                 color = "#A23B72", size = 1.2) +
    labs(
      title = "Distribution of Click-Through Rate (CTR)",
      subtitle = "Histogram with density overlay",
      x = "CTR",
      y = "Frequency"
    ) +
    theme_minimal() +
    theme(
      plot.title = element_text(face = "bold", size = 14),
      plot.subtitle = element_text(color = "gray40")
    )
  
  return(p)
}

# Usage:
# p1 <- plot_target_distribution(analysis_data)
# ggsave("visualizations/01_ctr_distribution.png", p1, width = 10, height = 6)

# ----------------------------------------------------------------------------------
# 1.2 Correlation Heatmap
# ----------------------------------------------------------------------------------
plot_correlation_matrix <- function(data) {
  # Select only numeric columns
  numeric_data <- data %>% select(where(is.numeric))
  
  # Calculate correlation matrix
  cor_matrix <- cor(numeric_data, use = "complete.obs")
  
  # Create correlation plot
  png("visualizations/02_correlation_heatmap.png", width = 1200, height = 1000, res = 120)
  corrplot(
    cor_matrix,
    method = "color",
    type = "upper",
    order = "hclust",
    tl.col = "black",
    tl.srt = 45,
    addCoef.col = "black",
    number.cex = 0.7,
    col = colorRampPalette(c("#A23B72", "white", "#2E86AB"))(200),
    title = "Feature Correlation Matrix",
    mar = c(0, 0, 2, 0)
  )
  dev.off()
  
  cat("Correlation heatmap saved to visualizations/02_correlation_heatmap.png\n")
}

# Usage:
# plot_correlation_matrix(analysis_data)

# ----------------------------------------------------------------------------------
# 1.3 Categorical Feature Distribution
# ----------------------------------------------------------------------------------
plot_categorical_features <- function(data) {
  # Time of Day
  p1 <- ggplot(data, aes(x = time_of_day, fill = time_of_day)) +
    geom_bar() +
    scale_fill_brewer(palette = "Set2") +
    labs(title = "Distribution by Time of Day", x = "", y = "Count") +
    theme(legend.position = "none")
  
  # Day of Week
  p2 <- ggplot(data, aes(x = day_of_week, fill = day_of_week)) +
    geom_bar() +
    scale_fill_brewer(palette = "Set3") +
    labs(title = "Distribution by Day of Week", x = "", y = "Count") +
    theme(legend.position = "none", axis.text.x = element_text(angle = 45, hjust = 1))
  
  # Age Group
  p3 <- ggplot(data, aes(x = age_group, fill = age_group)) +
    geom_bar() +
    scale_fill_brewer(palette = "Spectral") +
    labs(title = "Distribution by Age Group", x = "", y = "Count") +
    theme(legend.position = "none", axis.text.x = element_text(angle = 45, hjust = 1))
  
  # Combine plots
  combined <- grid.arrange(p1, p2, p3, ncol = 3)
  
  ggsave("visualizations/03_categorical_distributions.png", combined, 
         width = 15, height = 5)
  
  return(combined)
}

# Usage:
# plot_categorical_features(analysis_data)

# ----------------------------------------------------------------------------------
# 1.4 CTR by Time and Demographics
# ----------------------------------------------------------------------------------
plot_ctr_by_features <- function(data) {
  # CTR by Time of Day
  p1 <- ggplot(data, aes(x = time_of_day, y = CTR, fill = time_of_day)) +
    geom_boxplot(alpha = 0.7) +
    scale_fill_brewer(palette = "Set2") +
    labs(title = "CTR by Time of Day", x = "", y = "CTR") +
    theme(legend.position = "none")
  
  # CTR by Day of Week
  p2 <- ggplot(data, aes(x = day_of_week, y = CTR, fill = day_of_week)) +
    geom_boxplot(alpha = 0.7) +
    scale_fill_brewer(palette = "Set3") +
    labs(title = "CTR by Day of Week", x = "", y = "CTR") +
    theme(legend.position = "none", axis.text.x = element_text(angle = 45, hjust = 1))
  
  combined <- grid.arrange(p1, p2, ncol = 2)
  
  ggsave("visualizations/04_ctr_by_features.png", combined, 
         width = 12, height = 5)
  
  return(combined)
}

# Usage:
# plot_ctr_by_features(analysis_data)

# ==================================================================================
# SECTION 2: MODEL PERFORMANCE VISUALIZATION
# ==================================================================================

# ----------------------------------------------------------------------------------
# 2.1 Model Comparison Chart
# ----------------------------------------------------------------------------------
plot_model_comparison <- function() {
  model_results <- data.frame(
    Model = c("Seasonality\nLinear", "Random\nForest", 
              "Simple\nBoosting", "XGBoost\nTuned"),
    Train_RMSE = c(0.999, NA, NA, 0.086),
    Test_RMSE = c(NA, NA, NA, 0.059),
    Order = 1:4
  )
  
  # Reshape for plotting
  model_long <- model_results %>%
    select(-Order) %>%
    melt(id.vars = "Model", variable.name = "Set", value.name = "RMSE")
  
  p <- ggplot(model_long, aes(x = reorder(Model, c(1,2,3,4)), y = RMSE, fill = Set)) +
    geom_bar(stat = "identity", position = "dodge", alpha = 0.8) +
    scale_fill_manual(
      values = c("Train_RMSE" = "#2E86AB", "Test_RMSE" = "#A23B72"),
      labels = c("Training", "Testing")
    ) +
    labs(
      title = "Model Performance Comparison",
      subtitle = "RMSE across different modeling approaches",
      x = "Model",
      y = "RMSE",
      fill = "Dataset"
    ) +
    theme_minimal() +
    theme(
      plot.title = element_text(face = "bold", size = 14),
      legend.position = "top"
    )
  
  ggsave("visualizations/05_model_comparison.png", p, width = 10, height = 6)
  
  return(p)
}

# Usage:
# plot_model_comparison()

# ----------------------------------------------------------------------------------
# 2.2 Training vs Testing RMSE
# ----------------------------------------------------------------------------------
plot_train_test_rmse <- function() {
  data <- data.frame(
    Dataset = c("Training", "Testing"),
    RMSE = c(0.086, 0.059)
  )
  
  p <- ggplot(data, aes(x = Dataset, y = RMSE, fill = Dataset)) +
    geom_col(alpha = 0.8, width = 0.6) +
    geom_text(aes(label = sprintf("%.3f", RMSE)), 
              vjust = -0.5, size = 5, fontface = "bold") +
    scale_fill_manual(values = c("Training" = "#2E86AB", "Testing" = "#A23B72")) +
    labs(
      title = "Final XGBoost Model Performance",
      subtitle = "Lower test RMSE indicates good generalization",
      y = "Root Mean Square Error (RMSE)",
      x = ""
    ) +
    ylim(0, 0.1) +
    theme_minimal() +
    theme(
      plot.title = element_text(face = "bold", size = 14),
      legend.position = "none",
      panel.grid.major.x = element_blank()
    )
  
  ggsave("visualizations/06_final_model_rmse.png", p, width = 8, height = 6)
  
  return(p)
}

# Usage:
# plot_train_test_rmse()

# ==================================================================================
# SECTION 3: HYPERPARAMETER TUNING VISUALIZATION
# ==================================================================================

# ----------------------------------------------------------------------------------
# 3.1 Hyperparameter Impact Heatmap
# ----------------------------------------------------------------------------------
plot_hyperparameter_heatmap <- function(tuning_results) {
  # Example tuning results (replace with actual CV results)
  # tuning_results should have columns: eta, max_depth, rmse
  
  p <- ggplot(tuning_results, aes(x = factor(eta), y = factor(max_depth), fill = rmse)) +
    geom_tile(color = "white", size = 0.5) +
    scale_fill_gradient2(
      low = "#2E86AB", 
      mid = "white", 
      high = "#A23B72",
      midpoint = median(tuning_results$rmse)
    ) +
    geom_text(aes(label = sprintf("%.4f", rmse)), size = 3) +
    labs(
      title = "Hyperparameter Tuning Results",
      subtitle = "CV RMSE by learning rate and tree depth",
      x = "Learning Rate (eta)",
      y = "Max Depth",
      fill = "CV RMSE"
    ) +
    theme_minimal() +
    theme(
      plot.title = element_text(face = "bold", size = 14),
      legend.position = "right"
    )
  
  ggsave("visualizations/07_hyperparameter_heatmap.png", p, width = 10, height = 6)
  
  return(p)
}

# ----------------------------------------------------------------------------------
# 3.2 Learning Rate Impact
# ----------------------------------------------------------------------------------
plot_learning_rate_impact <- function(tuning_results) {
  p <- ggplot(tuning_results, aes(x = eta, y = rmse, color = factor(max_depth))) +
    geom_line(size = 1.2) +
    geom_point(size = 3) +
    scale_color_brewer(palette = "Set1", name = "Max Depth") +
    labs(
      title = "Impact of Learning Rate on Model Performance",
      x = "Learning Rate (eta)",
      y = "CV RMSE"
    ) +
    theme_minimal() +
    theme(
      plot.title = element_text(face = "bold", size = 14),
      legend.position = "right"
    )
  
  ggsave("visualizations/08_learning_rate_impact.png", p, width = 10, height = 6)
  
  return(p)
}

# ==================================================================================
# SECTION 4: FEATURE IMPORTANCE VISUALIZATION
# ==================================================================================

# ----------------------------------------------------------------------------------
# 4.1 Feature Importance Bar Plot
# ----------------------------------------------------------------------------------
plot_feature_importance <- function(importance_matrix, top_n = 15) {
  # Select top N features
  top_features <- head(importance_matrix, top_n)
  
  p <- ggplot(top_features, aes(x = reorder(Feature, Gain), y = Gain)) +
    geom_col(fill = "#2E86AB", alpha = 0.8) +
    coord_flip() +
    labs(
      title = paste("Top", top_n, "Most Important Features"),
      subtitle = "Based on XGBoost gain metric",
      x = "Feature",
      y = "Importance (Gain)"
    ) +
    theme_minimal() +
    theme(
      plot.title = element_text(face = "bold", size = 14)
    )
  
  ggsave("visualizations/09_feature_importance.png", p, width = 10, height = 8)
  
  return(p)
}

# ==================================================================================
# MASTER FUNCTION: GENERATE ALL VISUALIZATIONS
# ==================================================================================

generate_all_visualizations <- function(analysis_data, importance_matrix = NULL) {
  cat("Generating all visualizations...\n\n")
  
  # Create output directory
  dir.create("visualizations", showWarnings = FALSE)
  
  # EDA Visualizations
  cat("1. Creating target distribution plot...\n")
  plot_target_distribution(analysis_data)
  
  cat("2. Creating correlation heatmap...\n")
  plot_correlation_matrix(analysis_data)
  
  cat("3. Creating categorical feature distributions...\n")
  plot_categorical_features(analysis_data)
  
  cat("4. Creating CTR by features plots...\n")
  plot_ctr_by_features(analysis_data)
  
  # Model Performance
  cat("5. Creating model comparison chart...\n")
  plot_model_comparison()
  
  cat("6. Creating train-test RMSE comparison...\n")
  plot_train_test_rmse()
  
  # Feature Importance
  if (!is.null(importance_matrix)) {
    cat("7. Creating feature importance plot...\n")
    plot_feature_importance(importance_matrix)
  }
  
  cat("\n✅ All visualizations generated successfully!\n")
  cat("Check the 'visualizations' folder for outputs.\n")
}

# ==================================================================================
# USAGE EXAMPLE
# ==================================================================================

# Uncomment to run:
# 
# # Load your data
# analysis_data <- read.csv("~/Downloads/predicting-clicks/analysis_data.csv")
# 
# # If you have feature importance from XGBoost:
# importance_matrix <- read.csv("feature_importance.csv")
# 
# # Generate all plots
# generate_all_visualizations(analysis_data, importance_matrix)

cat("\n📊 Visualization script loaded successfully!\n")
cat("Use generate_all_visualizations() to create all plots.\n")