dataAnalysis.tex

\documentclass{article}
\usepackage{amsmath,amssymb}
\usepackage{hyperref}
\usepackage{bm}

\newcommand{\yti}{Y_{Ti}}
\newcommand{\yci}{Y_{Ci}}
\newcommand{\uti}{U_{Ti}}
\newcommand{\uci}{U_{Ci}}
\newcommand{\etat}{\eta_T}
\newcommand{\etati}{\eta_{Ti}}

\newcommand{\mti}{\bar{m}_{Ti}}
\newcommand{\byt}{\bm{Y_T}}
\newcommand{\byc}{\bm{Y_C}}
\newcommand{\bmt}{\bm{\bar{m}_T}}
\newcommand{\bmi}{\bm{m}_i}
\newcommand{\bsi}{\bm{s}_i}

\newcommand{\EE}{\mathbb{E}}

\title{Data Analysis for 'The Role of Mastery Learning'}




\begin{document}
This document includes code to produce all of the results and run all
of the models reported in ``The Role of Mastery Learning in
Intelligent Tutoring Systems: Principal Stratification on a Latent Variable.''

The auxilliary files sourced here are available at our github repository,
\url{https://github.com/adamSales/ctaiAdvance}.

First, load in and transform the (pre-imputed) data:
\begin{kframe}
\begin{alltt}
\hlkwd{load}\hlstd{(}\hlstr{'data/HSdata.RData'}\hlstd{)}
\hlkwd{load}\hlstd{(}\hlstr{'data/advanceData.RData'}\hlstd{)}
\end{alltt}
\end{kframe}
We'll use the \texttt{R} package \texttt{rstan} to run the models:
\begin{kframe}
\begin{alltt}
\hlkwd{library}\hlstd{(rstan)}
\hlkwd{rstan_options}\hlstd{(}\hlkwc{auto_write} \hlstd{=} \hlnum{TRUE}\hlstd{)}
\hlkwd{options}\hlstd{(}\hlkwc{mc.cores} \hlstd{= parallel}\hlopt{::}\hlkwd{detectCores}\hlstd{())}
\end{alltt}
\end{kframe}

\section{Data Description (Section \ref{sec:data})}

This code produces the missigness information from Table 1, summarizing the student level data:
\begin{kframe}
\begin{alltt}
\hlstd{miss} \hlkwb{<-} \hlkwa{NULL}
\hlkwa{for}\hlstd{(i} \hlkwa{in} \hlkwd{c}\hlstd{(}\hlstr{'race'}\hlstd{,}\hlstr{'sex'}\hlstd{,}\hlstr{'spec'}\hlstd{,}\hlstr{'xirt'}\hlstd{)) miss} \hlkwb{<-} \hlkwd{rbind}\hlstd{(miss,}
 \hlkwd{c}\hlstd{(}\hlkwd{sum}\hlstd{(}\hlkwd{is.na}\hlstd{(covs[[i]])),}\hlkwd{mean}\hlstd{(}\hlkwd{is.na}\hlstd{(covs[[i]])),error[i,}\hlstr{'error'}\hlstd{]))}
\hlstd{miss} \hlkwb{<-} \hlkwd{as.data.frame}\hlstd{(miss)}
\hlstd{miss}\hlopt{$}\hlstd{`Error Type`} \hlkwb{<-} \hlkwd{c}\hlstd{(}\hlstr{'PFC'}\hlstd{,}\hlstr{'PFC'}\hlstd{,}\hlstr{'PFC'}\hlstd{,}\hlstr{'SRMSE'}\hlstd{)}
\hlkwd{rownames}\hlstd{(miss)} \hlkwb{<-} \hlkwd{c}\hlstd{(}\hlstr{'Race/Ethnicity'}\hlstd{,}\hlstr{'Sex'}\hlstd{,}\hlstr{'Special Education'}\hlstd{,}\hlstr{'Pretest'}\hlstd{)}
\hlkwd{names}\hlstd{(miss)[}\hlnum{1}\hlopt{:}\hlnum{3}\hlstd{]} \hlkwb{<-} \hlkwd{c}\hlstd{(}\hlstr{'# Missing'}\hlstd{,}\hlstr{'% Missing'}\hlstd{,}\hlstr{'Imputation Error'}\hlstd{)}
\hlstd{miss[,}\hlnum{2}\hlstd{]} \hlkwb{<-} \hlkwd{as.integer}\hlstd{(}\hlkwd{round}\hlstd{(miss[,}\hlnum{2}\hlstd{]}\hlopt{*}\hlnum{100}\hlstd{))}
\hlstd{miss[,}\hlnum{1}\hlstd{]} \hlkwb{<-} \hlkwd{as.integer}\hlstd{(miss[,}\hlnum{1}\hlstd{])}
\hlstd{miss[}\hlstr{'Pretest'}\hlstd{,}\hlstr{'Imputation Error'}\hlstd{]} \hlkwb{<-} \hlkwd{sqrt}\hlstd{(miss[}\hlstr{'Pretest'}\hlstd{,}\hlstr{'Imputation Error'}\hlstd{])}\hlopt{/}\hlkwd{sd}\hlstd{(covs}\hlopt{$}\hlstd{xirt,}\hlkwc{na.rm}\hlstd{=}\hlnum{TRUE}\hlstd{)}


\hlkwd{print}\hlstd{(xtable}\hlopt{::}\hlkwd{xtable}\hlstd{(miss))}
\end{alltt}
\end{kframe}% latex table generated in R 3.4.2 by xtable 1.8-2 package
% Tue Feb 06 18:23:02 2018
\begin{table}[ht]
\centering
\begin{tabular}{rrrrl}
  \hline
 & \# Missing & \% Missing & Imputation Error & Error Type \\ 
  \hline
Race/Ethnicity & 1071 &   8 & 0.23 & PFC \\ 
  Sex & 526 &   4 & 0.35 & PFC \\ 
  Special Education & 199 &   1 & 0.11 & PFC \\ 
  Pretest & 2367 &  18 & 0.20 & SRMSE \\ 
   \hline
\end{tabular}
\end{table}

This code produces the covariate balance information:
\begin{kframe}
\begin{alltt}
\hlstd{covBal} \hlkwb{<-} \hlkwa{NULL}
\hlkwa{for}\hlstd{(i} \hlkwa{in} \hlkwd{c}\hlstd{(}\hlstr{'race'}\hlstd{,}\hlstr{'sex'}\hlstd{,}\hlstr{'spec'}\hlstd{))\{}
    \hlstd{covBal} \hlkwb{<-} \hlkwd{rbind}\hlstd{(covBal,}\hlkwd{c}\hlstd{(i,}\hlnum{NA}\hlstd{,}\hlnum{NA}\hlstd{,}\hlnum{NA}\hlstd{,}\hlnum{NA}\hlstd{))}

    \hlkwa{for}\hlstd{(ll} \hlkwa{in} \hlkwd{levels}\hlstd{(dat[[i]]))\{}
        \hlstd{covBal} \hlkwb{<-} \hlkwd{rbind}\hlstd{(covBal,}\hlkwd{c}\hlstd{(}\hlnum{NA}\hlstd{,ll,}\hlkwd{round}\hlstd{(}\hlkwd{c}\hlstd{(}\hlkwd{mean}\hlstd{(dat[[i]]}\hlopt{==}\hlstd{ll),}\hlkwd{mean}\hlstd{(dat[[i]][dat}\hlopt{$}\hlstd{treatment}\hlopt{==}\hlnum{1}\hlstd{]}\hlopt{==}\hlstd{ll),}\hlkwd{mean}\hlstd{(dat[[i]][dat}\hlopt{$}\hlstd{treatment}\hlopt{==}\hlnum{0}\hlstd{]}\hlopt{==}\hlstd{ll)),}\hlnum{2}\hlstd{)))}

    \hlstd{\}}
\hlstd{\}}
\hlkwd{colnames}\hlstd{(covBal)} \hlkwb{<-} \hlkwd{c}\hlstd{(}\hlstr{'Covariate'}\hlstd{,}\hlstr{'Category'}\hlstd{,}\hlstr{'Overall Percent'}\hlstd{,}\hlstr{'Percent of Treated'}\hlstd{,}\hlstr{'Percent of Control'}\hlstd{)}
\hlkwd{print}\hlstd{(}\hlkwd{xtable}\hlstd{(covBal),}\hlkwc{floating}\hlstd{=}\hlnum{FALSE}\hlstd{,}\hlkwc{include.rownames}\hlstd{=}\hlnum{FALSE}\hlstd{)}
\end{alltt}
\end{kframe}% latex table generated in R 3.4.2 by xtable 1.8-2 package
% Tue Feb 06 18:23:03 2018
\begin{tabular}{lllll}
  \hline
Covariate & Category & Overall Percent & Percent of Treated & Percent of Control \\ 
  \hline
race &  &  &  &  \\ 
   & WhiteAsian & 0.49 & 0.48 & 0.49 \\ 
   & BlackMulti & 0.33 & 0.3 & 0.35 \\ 
   & HispAIAN & 0.18 & 0.22 & 0.16 \\ 
  sex &  &  &  &  \\ 
   & F & 0.5 & 0.49 & 0.5 \\ 
   & M & 0.5 & 0.51 & 0.5 \\ 
  spec &  &  &  &  \\ 
   & typical & 0.9 & 0.89 & 0.9 \\ 
   & speced & 0.07 & 0.08 & 0.07 \\ 
   & gifted & 0.03 & 0.04 & 0.03 \\ 
   \hline
\end{tabular}

The overall p-value for balance is:
\begin{knitrout}
\definecolor{shadecolor}{rgb}{0.969, 0.969, 0.969}\color{fgcolor}\begin{kframe}
\begin{alltt}
\hlkwd{library}\hlstd{(RItools)} \hlcom{## using development version}
\hlstd{balMod} \hlkwb{<-} \hlkwd{balanceTest}\hlstd{(treatment}\hlopt{~}\hlkwd{poly}\hlstd{(xirt,}\hlnum{2}\hlstd{)}\hlopt{+}\hlstd{spec}\hlopt{+}\hlstd{race}\hlopt{+}\hlstd{sex}\hlopt{+}\hlkwd{strata}\hlstd{(pair)}\hlopt{+}\hlkwd{cluster}\hlstd{(schoolid2),}\hlkwc{data}\hlstd{=dat,}\hlkwc{report}\hlstd{=}\hlstr{'chisquare.test'}\hlstd{)}
\hlkwd{print}\hlstd{(balMod}\hlopt{$}\hlstd{overall[}\hlstr{'pair'}\hlstd{,])}
\end{alltt}
\begin{verbatim}
## chisquare        df   p.value 
##    8.4464    7.0000    0.2949
\end{verbatim}
\end{kframe}
\end{knitrout}

\section{PS Model with $\bar{m}_T$}

Here we estimate the model in Section
\ref{sec:principalStratification} stratifying on $\bar{m}_T$.

First, we create the datasets:
\begin{kframe}
\begin{alltt}
\hlkwd{source}\hlstd{(}\hlstr{'R/prelimMbar.r'}\hlstd{)}
\end{alltt}
\end{kframe}

The model is encoded in the file \texttt{psmodObs.stan}.
It may be summarized as follows.
The model for $\bar{m}_T$ is:
\begin{equation}\label{eq:mbarUsage}
\bar{m}_{Ti}=\alpha^U_s+\bm{x}_i^T\bm{\beta^U}+\epsilon^{Ui}_i+\epsilon^{Ut}_{t[i]}
\end{equation}
where $\alpha^U_s$ is a separate intercept for each state, and $\bm{x}_i$ is a vector of covariates: dummy variables for
racial/ethnic category, a dummy variable for sex, dummy variables for
special education category, and linear and quadratic terms for
pretest.
The normally-distributed errors $\epsilon^{Ui}$ and $\epsilon^{Ut}_{t[i]}$ vary at the
individual and teacher levels.

The model for $Y$ is
\begin{equation}
Y_i=\alpha^Y_p+\bm{x}_i^T\bm{\beta^Y}+a_1\bar{m}_{Ti}+Z_i(b_0+b_1*\bar{m}_{Ti})+\epsilon^{Yi}_i+\epsilon^{Yt}_{t[i]}+\epsilon^{Ys}_{s[i]}
\end{equation}
where $\alpha^Y_p$ is a separate intercept for each randomization
block $p$, $Z_i$ is a dummy variable for treatment status,
$\epsilon^{Ys}_{s[i]}$ is a normally distributed error at the school
level, and the rest of the variables are analogous to those in
(\ref{eq:mbarUsage}).
We run the model with the \texttt{stan} command from \texttt{rstan}: