Notes.tex

\documentclass[a4paper, 11pt]{article}
\usepackage{graphicx}\usepackage{amsmath}\usepackage{float}
\usepackage[inner=1in, top=1in, bottom=1in]{geometry}
\usepackage{amsfonts}
\usepackage{color,soul}
\usepackage{framed}
\usepackage{hyperref}
\usepackage{listings}
\usepackage{gensymb}
\DeclareMathOperator{\Cor}{Cor}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\STD}{STD}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\E}{E}
\DeclareMathOperator{\Tr}{Tr}
\DeclareMathOperator{\diag}{diag}
\DeclareMathOperator{\x}{\boldsymbol{x}}
\DeclareMathOperator{\X}{\boldsymbol{X}}
\DeclareMathOperator{\y}{\boldsymbol{y}}
\DeclareMathOperator{\z}{\boldsymbol{z}}
\DeclareMathOperator{\f}{\boldsymbol{f}}
\DeclareMathOperator{\bmu}{\boldsymbol{\mu}}
\DeclareMathOperator{\bSigma}{\boldsymbol{\Sigma}}
\DeclareMathOperator{\pa}{\boldsymbol{\theta}}

\begin{document}

\title{Conditional Distributions of a Gaussian Mixture Model}
\author{P.L.Green\\
School of Engineering\\
University of Liverpool\\
Liverpool, L69 7ZF\\
United Kingdom\\
\\
\href{mailto:p.l.green@liverpool.ac.uk}{p.l.green@liverpool.ac.uk} \\
\url{https://www.liverpool.ac.uk/engineering/staff/peter-green/}
}
\maketitle

\section{Introduction}
Gaussian Mixture Models are a classic clustering technique, that can easily be generalised to, for example, semi-supervised learning. Sometimes we need to compute closed-form expressions for the conditional distributions. Finding this a bit more difficult that expected, these notes and code were created as a way of making sure that I'd done it properly...  

\section{Conditional of a Gaussian Mixture Model}
Say we have the following Gaussian Mixture Model:

\begin{equation}
	p(\x_1, \x_2) = 
    \sum_c \Pr(c)
    \mathcal{N}\left(
		\left(
		\begin{array}{c}
			\x_1 \\
			\x_2 \\
		\end{array}
		\right);
		\left(
		\begin{array}{c}
			\bmu_1^{(c)} \\
			\bmu_2^{(c)} \\
		\end{array}
		\right),
		\left[
		\begin{array}{cc}
			\bSigma_{1,1}^{(c)} & \bSigma_{1,2}^{(c)} \\
			\bSigma_{2,1}^{(c)} & \bSigma_{2,2}^{(c)} \\
		\end{array}
		\right]
	\right)
\end{equation}
where $\x_1 \in \mathbb{R}^{D_1}$, $\x_2 \in \mathbb{R}^{D_2}$, $c$ indexes each Gaussian in the mixture and $\Pr(c)$ is the mixture proportion associated with the $c$th Gaussian. Our aim is to derive an expression for $p(\x_1 | \x_2)$. \\

We begin by noting that

\begin{equation}
	p(\x_1| \x_2) = \sum_c \Pr(c| \x_2) \mathcal{N}\left(
		\x_1; \bmu_{1|2}^{(c)}, \bSigma_{1|2}^{(c)}
	\right)
    \label{eq:GMM_cond}
\end{equation}
where, using standard properties of Gaussian distributions, we know that:

\begin{equation}
	\bmu_{1|2}^{(c)} = \bmu_{1}^{(c)} + \bSigma_{1,1}^{(c)} \left(\bSigma^{(c)}_{2,2}\right)^{-1} (\x_2 - \bmu_2^{(c)})
\end{equation}
and

\begin{equation}
	\bSigma_{1|2}^{(c)} = \bSigma_{1,1}^{(c)} - \bSigma_{1,2}^{(c)} \left(\bSigma^{(c)}_{2,2}\right)^{-1} \bSigma^{(c)}_{2,1}
\end{equation}
Note that the mixture proportions in equation (\ref{eq:GMM_cond}) are now conditional on $\x_2$ and that, in general, we cannot say that $\Pr(c)$ will be equal to $\Pr(c | \x_2)$. To evaluate $\Pr(c | \x_2)$ we use Bayes' theorem to obtain:

\begin{equation}
    \Pr(c | \x_2) = \frac{p(\x_2 | c) \Pr(c)}{\sum_{c'} p(\x_2 | c') \Pr(c')}
\end{equation}

\begin{equation}
    = \frac{ \mathcal{N}(\x_2 | \bmu_2^{(c)}, \bSigma_{2,2}^{(c)}) \Pr(c)}{\sum_{c'} \mathcal{N}(\x_2 | \bmu_2^{(c')}, \bSigma_{2,2}^{(c')}) \Pr(c')}
\end{equation}
Example code is implemented in Python 3. 

\end{document}