02-NumDNN-LinearLeastSquares.tex

\documentclass[12pt,fleqn,handout]{beamer}

\input{beamerStyle.tex}
\input{abbrv.tex}

\title{Linear Least-Squares}
\subtitle{Numerical Methods for Deep Learning}
\date{}

\begin{document}

\makebeamertitle



\section{Least-Squares Regression} 
\label{sec:least_squares_regression}

\begin{frame}
	\frametitle{Learning Objective: Linear Least-Squares}
	
	In this module we review computational methods for linear least-squares problems, which play a role in almost any learning tasks (including deep learning).
	
	\bigskip
	
	Learning tasks:
	\begin{itemize}
		\item image classification (supervised)
		\item image inpainting (semi-supervised) 
		\item \ldots
	\end{itemize}
	
	\bigskip
	
	Numerical methods:
	\begin{itemize}
		\item singular value decomposition
		\item steepest descent
		\item conjugate gradient method
	\end{itemize}
\end{frame}

\subsection{Introduction} % (fold)
\label{sub:introduction}

\begin{frame}\frametitle{Reminder: Supervised Learning Problem}

Given examples (inputs)
$$ \bfY = \left[\bfy_1,  \bfy_2 , \cdots, \bfy_n \right] \in\R^{n_f\times n}$$
and labels (outputs)
$$ \bfC = \left[ \bfc_1,  \bfc_2, \cdots,  \bfc_n \right] \in\R^{n_c\times n},$$

find a classification/prediction function $f(\cdot,\bftheta)$, i.e., 

$$
f(\bfy_j,\bftheta) \approx \bfc_j, \quad j=1,\ldots,n.
$$

\end{frame}


\begin{frame}\frametitle{Regression and Least-Squares}

Simplest option, a linear model with $\bftheta = (\bfW,\bfb)$ and
$$ f(\bfY,\bfW,\bfb) =  \bfW \bfY + \bfb \bfe_n^\top \approx \bfC $$
\begin{itemize}
	\item $\bfW \in \R^{n_c \times n_f}$ are \emph{weights}
	\item $\bfb \in \R^{n_c}$ are \emph{biases}
	\item $\bfe_n \in \R{^n}$ is a vector of ones
\end{itemize} 
\pause
Equivalent notation:
$$f(\bfY,\bfW,\bfb) = \begin{pmatrix} \bfW & \bfb \end{pmatrix} \begin{pmatrix} \bfY \\ \bfe_n^\top \end{pmatrix} \approx \bfC$$

\pause

Problem may not have a solution, or may have infinite solutions (when?).
Solve through optimization
 
$$ \min_{\bfW} \hf \|\bfW \bfY - \bfC\|^2_F $$

(Frobenius norm: $\|\bfA\|_F^2 = {\rm trace}(\bfA^{\top} \bfA) = \sum_{i,j} \bfA_{i,j}^2. $)

\end{frame}

\begin{frame}
	\frametitle{Remark: Relation to Least-Squares}
	
	Consider the regression problem
	$$ \min_{\bfW} \hf \|\bfW \bfY - \bfC\|^2_F. $$
	
	It is easy to see that this is equivalent to
	$$ \min_{\bfW} \hf \| \bfY^\top \bfW^\top - \bfC^\top\|^2_F, $$
	
	which can be solved separately for each row in $\bfW$
	$$
	\bfW(j,:)^\top = \argmin_{\bfw} \hf \| \bfY^\top \bfw - \bfC(j,:)^\top\|^2_F.
	$$
	
	Notation: Let $\bfA = \bfY^\top$ and $\bfX = \bfW^\top$ (easy to add bias here), we solve
	$$
		\min_\bfX \hf \| \bfA \bfX - \bfC^\top \|^2_F
	$$
		
\end{frame} 

\begin{frame}
	\frametitle{Detour: Image Inpainting (Semi-Supervised)}
	
	Image inpainting: Estimate an image function $f : [0,1]^2 \to \R$ from partial observations.
	
	First: Discretize the image on a grid with $n = n_x \cdot n_y$ pixels. We obtain $\bff \in \R^n$. 
	
	\bigskip
	\pause
	
	The input data is given by $\bfc \in \R^{k}$ with $k<n$ where 
	\begin{equation*}
		\bfd_i = f(\bfx_i) + \bfepsilon_i, \quad i=1,\ldots,k,
	\end{equation*}
	where $\bfx_i$ are points in $[0,1]^2$, and $\bfepsilon \sim \mathcal{N}(0,\sigma\bfI)$.
	
	\bigskip
	
	Write the inpainting problem as a linear least-squares problem
	\begin{equation*}
		\min_{\bff} \frac{1}{2\sigma} \| \bfA \bff - \bfc \|^2.
	\end{equation*} 
	Question: What is $\bfA$? How good will this approach be?
	
	
\end{frame}

% subsection introduction (end)


\begin{frame}\frametitle{Optimality Conditions for Least-Squares}

To minimize a function need to differentiate and equate to $0$

$$ {\frac {\partial \left( \hf \|\bfA\bfX - \bfC^\top\|^2_F \right)}{\partial \bfX}} = 0 $$

Compute the derivatives in three steps


\begin{enumerate}
\item $$ {\frac {\partial \left( \hf \|\bfR \|^2_F \right)}{\partial \bfR}} = ??? $$

\item $$ {\frac {\partial \left( \bfA \bfX \right)}{\partial \bfX}} = ??? $$

\item Use chain rule

\end{enumerate}


\end{frame} \begin{frame}\frametitle{Least-Squares: Normal Equations}

The necessary and sufficient optimality conditions for the least-squares problem are
$$ {\frac {\partial \left( \hf \|\bfA\bfX - \bfC^\top\|^2_F \right)}{\partial \bfX}} = 
\bfA^\top(\bfA \bfX - \bfC^\top)=0 $$

Reorganize to obtain the {\bf normal equations}

$$ \bfX =  (\bfA^\top \bfA)^{-1}(\bfA^\top \bfC^\top). $$

Here, $\bfA^\top \bfA \in \R^{n_f \times n_f}$ must be invertible, i.e., 
\begin{itemize}
\item sufficient amount of data $(n>n_f)$
\item data is linearly independent
\end{itemize}

\end{frame}

% section coding (end)
\begin{frame}
	\frametitle{Coding: Least-Squares for Classification}
	
	\begin{enumerate}
		\item Write a code for solving
		$$
		\min_{\bfW,\bfb} \hf\| \bfW\bfY + \bfb \bfe_n^\top - \bfC \|^2
		$$
		and apply it to the MNIST test data from
		\begin{center}
			\texttt{http://yann.lecun.com/exdb/mnist/}
		\end{center}
		
		\item Solve the problem using the normal equations derived above.
		
		\item Use optimal weights to predict labels for test data. How well does your solution generalize?
		
		\item Visualize the rows of $\bfW$ as images.
		
	\end{enumerate}
	
	\bigskip
	
\end{frame}
\section{Regularization} 
\label{sec:bias_variance}

\begin{frame}\frametitle{Ill-posedness and the SVD}

If the data is linearly dependent or close to be linearly dependent, least-squares problem gives no good solution~\cite{Hansen1998,Vogel2002,Hansen2010}.

Understanding can be gained by the \emph{Singular Value Decomposition} (SVD) (e.g.,~\cite[Ch. 8]{AscherGreif2011})

$$ \bfA = \bfU \bfSigma \bfV^{\top} $$

where $\bfU \in \R^{n_f\times n_f}, \bfV\in\R^{n_f \times n}$ satisfy
$$  \bfU^{\top} \bfU = \bfI, \quad \text{ and } \quad \bfV^{\top} \bfV = \bfI $$
Diagonal of $\bfSigma$ contains the singular values $\sigma_1 \geq ... \sigma_{n_f} \geq 0 $
$$ \bfSigma = \begin{pmatrix} \sigma_1  &  &  \\ &  \ddots &  \\ &  &  \sigma_{n_f} \end{pmatrix} $$

\end{frame}



\begin{frame}\frametitle{Ill-posedness and Regularization}

Important is the \emph{effective rank}: If $\sigma_j \ll \sigma_k$ for all
$j\geq k$, then the effective rank of the problem is $k$.

\bigskip

If $k < n_f$, the least squares problem is ill-posed, i.e., solution does not exist or is unstable.

Small perturbations
in $\bfC$ or $\bfA$ yield large perturbations in $\bfX$

\bigskip

Solve regularized problem: For some $\lambda>0$ and matrix $\bfG$
$$ \min_\bfX \hf \|\bfA\bfX - \bfC^\top\|_F^2 + {\frac {\lambda}2} \|\bfG \bfX\|_F^2 $$

{
{\em Exercise: solve the regularized least-squares problem}
}
\pause


$$ \bfX = (\bfA^{\top} \bfA + \lambda \bfG^\top\bfG)^{-1} \bfA^{\top} \bfC^\top $$
\end{frame}

\begin{frame}\frametitle{The Bias-Variance Decomposition}

Assume $\bfC^\top = \bfA \bfX_{\rm true}  + \bfepsilon$, $\bfepsilon \sim\mathcal{N}(0,\sigma \bfI)$,  $\lambda>0$ fixed, $\bfG=\bfI$.

Then setting  $\bfA^{\dag}_{\lambda} = (\bfA^{\top} \bfA + \lambda \bfI)^{-1}$
\begin{align*}
	\bfX - \bfX_{\rm true} & = \bfA^{\dag}_{\lambda} \bfA^{\top} \bfC^\top - \bfX_{\rm true} \\
	                      & = \left(\bfA^{\dag}_{\lambda} \bfA^{\top} \bfA - \bfI \right)  \bfX_{\rm true} +
	\bfA^{\dag}_{\lambda} \bfA^{\top} \bfepsilon \\
	                     & = -\lambda \bfA^{\dag}_{\lambda}\bfX_{\rm true}+
	\bfA^{\dag}_{\lambda} \bfA^{\top} \bfepsilon
\end{align*}

\pause

Error depends on $\bfepsilon$ $\leadsto$ take expectation
\begin{align*}
 {\mathbb E} \|\bfX - \bfX_{\rm true}\|_F^2 &= {\mathbb E}
\|\bfA^{\dag}_{\lambda} \bfA^{\top} \bfepsilon -\lambda \bfA^{\dag}_{\lambda}\bfX_{\rm true} \|_F^2   \\
&=\overbrace{ \lambda^2 \|\bfA^{\dag}_{\lambda}\bfX_{\rm true}\|_F^2}^{\rm \|bias\|_F^2} +
\overbrace{\sigma^2 {\rm trace}\left( \bfA \bfA^{\dag^T}_{\lambda} \bfA^{\dag}_{\lambda} \bfA^{\top} \right)}^{\rm variance}
\end{align*}

\pause
\begin{center}
	\textcolor{red}{Take home: No such thing as exact recovery!}	
\end{center}


\end{frame}


% \begin{frame}\frametitle{Next Time}
%
% Solving large-scale least-squares problems.
%
%
% \bigskip
%
% Watch: \tt{https://www.youtube.com/watch?v=eAYohMUpPMA}
% \begin{center}
% 	\includegraphics[width=.8\textwidth]{youtubeCG}
% \end{center}
%
%
% \end{frame}


\section{Iterative Methods} % (fold)
\label{sec:iterative_methods}
\begin{frame}\frametitle{Iterative Solvers for Least-Squares Regression}
	So far: Given $\bfY\in \R^{n_f\times n}$ and $\bfC\in\R^{n_c\times n}$, solve
	$$
	\min_{\bfX}  \frac12\left\|  \bfA   \bfX  -  \bfC^\top \right\|_F^2
	$$
	directly using $\bfX^* = (\bfA^{\top} \bfA)^{-1}\bfA^{\top} \bfC^\top$. Here
	$$
		\bfA = \begin{pmatrix}
			\bfY^\top && \bfe_n
		\end{pmatrix}, \quad \text{ and } \quad 
		\bfX = \bfW^\top \in \R^{(n_f+1)\times n_c}.
	$$
	Problems:
	\begin{enumerate}
		\item Generating  $\bfA^{\top}\bfA$ and solving normal equations is too costly for large-scale problems.
		\item Exact solution not useful when problem is ill-posed $\leadsto$ add explicit regularization or do so implicitly by early stopping.
	\end{enumerate} 
	
	\pause
Iterative methods that avoid working with $\bfA^{\top}\bfA$ 
\begin{itemize}
\item Steepest descent 
\item Conjugate gradient for least-squares (CGLS)
\end{itemize}
Excellent references: Numerical Optimization~\cite{NocedalWright2006}, iterative linear algebra~\cite{Saad2003}, general introduction \cite{AscherGreif2011}
\end{frame}

\begin{frame}\frametitle{Iterative Methods}


General idea - obtain a sequence $\bfX_1, \ldots,\bfX_j,\ldots$
that converges to least-squares solution $\bfX^*$

$$
	\bfX_j \longrightarrow \bfX^*, \quad \text{ for } \quad j \to \infty.
$$

\bigskip

How fast does the sequence converge? Assume
$$
\|\bfX_{j+1} - \bfX^* \| < \gamma_j \|\bfX_{j} - \bfX^* \|
$$ 
where all $\gamma_j < 1$. Then

\begin{itemize}
\item If $\gamma_j$ is bounded away from $0$ and 1 the convergence is linear
\item If $\gamma_j \rightarrow 0$ the convergence is superlinear
\item If $\gamma_j \rightarrow 1$ the convergence is sublinear
\end{itemize}

The sequence converges quadratically if $\gamma_j$ is bounded away from 0 and 1 and 
$$ \|\bfX_{j+1} - \bfX^* \| < \gamma_j \|\bfX_{j} - \bfX^* \|^2 $$ 
\end{frame}

\input{slides/sd.tex}

\begin{frame}\frametitle{Steepest Descent for Least-Squares}

Consider now
$$ \phi(\bfx) = \hf \|\bfA \bfx - \bfc\|^2 \quad \text{ with } \quad \grad_{\bfx} \phi(\bfx) = \bfA^{\top}(\bfA \bfx -  \bfc).$$

\bigskip

Steepest descent direction is  $\bfd_j = \bfA^{\top}(\bfc - \bfA \bfx_j)$ and
$$ \bfx_{j+1} = \bfx_j + \alpha_j \bfd_j $$

How to choose $\alpha_j$? 
\pause
Idea: Minimize $\phi$ along direction $\bfd_j$
$$ 
\alpha_j = \argmin_\alpha \phi(\bfx_j + \alpha \bfd_j) = 
 \argmin_\alpha \hf \|\alpha \bfA \bfd_j - \bfr_j\|^2
$$
with residual $\bfr_j = \bfc - \bfA \bfx_j $.

\bigskip
\pause

This leads to simple quadratic equation in 1D whose solution is
 
  
$$ \alpha_j = {\frac { \bfr_j^{\top}\bfA\bfd_j}{\|\bfA \bfd_j\|^2}} $$


\end{frame}


\begin{frame}\frametitle{Algorithm: Steepest Descent for Least-Squares}


for $j=1,\ldots $
\begin{itemize}
\item Compute residual $\bfr_j = \bfc - \bfA \bfx_j$
\item Compute the SD direction $\bfd_j = \bfA^{\top}\bfr_j$
\item Compute step size  $\alpha_j = {\frac { \bfr_j^{\top}\bfA\bfd_j }{\|\bfA \bfd_j\|^2}}$
\item Take the step $\bfx_{j+1} = \bfx_j + \alpha_j \bfd_j$
\end{itemize}

\pause

Converges linearly, i.e., 
$$
	\| \bfX_{j+1} - \bfX^*\| < \gamma \|\bfX_j - \bfX^* \| \quad \text{ with } \quad \gamma \approx \left| {\frac {\kappa-1}{\kappa+1}} \right|
$$

Here, $\kappa$ depends on condition number of $\bfA$, i.e., 
$$ \kappa \approx {\frac {\sigma_{\bf max}^2}{\sigma_{\bf min}^2}} $$


Can be painfully slow for ill-conditioned problems

\end{frame}


\begin{frame}\frametitle{Accelerating Steepest Descent: Post-Conditioning}

\begin{columns}
	\column{.7\textwidth}
	
	Idea: Improve convergence by transforming the problem
	 $$ \phi(\bfx) = \hf \|\bfA \bfS \bfS^{-1}\bfx - \bfc \|^2$$
	
	Here: $\bfS$ is invertible

	Solve in two steps:
	\begin{enumerate}
		\item Set $\bfz = \bfS^{-1}\bfx$ and compute
	 $$ \bfz^*  = \argmin_{\bfz} \hf \|\bfA \bfS \bfz - \bfc \|^2$$
	 	\item Then $\bfx = \bfS \bfz$.
	\end{enumerate} 

	 \bigskip

	 Pick $\bfS$ such that $\bfA \bfS$ is better conditioned.

	\column{.3\textwidth}
	\pause
	\begin{tabular}{@{}c@{}}
		original problem: \\
		\includegraphics[width=35mm]{convSD2DQuadratic}\\
		post-conditioned:\\
		\includegraphics[width=35mm]{convSD2DQuadraticGood} \\
	\end{tabular}
\end{columns}

\end{frame}



\begin{frame}\frametitle{Exercise: Steepest Descent for Least-Squares}

Goal: Program steepest descent and solve a simple problem.

\bigskip

To verify your code generate data using
$$ \bfc = \bfA \bfx_{\rm true} + \bfepsilon. $$
where $\bfepsilon$ is random with zero mean and standard deviation $0.1$ and
$$
 \bfY = \begin{pmatrix}  1 & 1+a \\ 1 & 1+2a \\ 1 & 1+3a
	\end{pmatrix} 
	\quad  \text{ and } \quad
	 \bfx_{\rm true} = \begin{pmatrix} 1\\ 1.2 \end{pmatrix}.
$$

\bigskip

Plot errors $\|\bfx_j - \bfx_{\rm true}\|$ for $j=1,\dots$ and $a \in \{1, 10^{-2}, 10^{-5}\}$.

\end{frame}



\begin{frame}\frametitle{Conjugate Gradient Method for Least-Squares}

CG is designed to solve quadratic optimization problems 
$$\min_\bfx \hf \bfx^{\top} \bfH \bfx - \bfb^{\top} \bfx$$
with $\bfH$ symmetric positive definite. In our case

$$ \argmin_\bfx \hf \|\bfA\bfx - \bfc\|^2 = \argmin_\bfx \hf \bfx^{\top} \underbrace{\bfA^{\top}\bfA}_{=\bfH} \bfx - \underbrace{\bfc^{\top}\bfA}_{=\bfb^\top}\bfx $$

\bigskip
 

CG improves over SD by using previous step (not a memory-less method) and constructing a basis for the solution.

\bigskip

Facts:
\begin{itemize}
	\item terminates after at most $n$ steps (in exact arithmetic)
	\item good solutions for $j\ll n$ 
	\item convergence
	$ \gamma_j \approx \left| {\frac {\sqrt{\kappa}-1}{\sqrt{\kappa}+1}} \right|^j $
\end{itemize}


\end{frame}

\begin{frame}[fragile]
	\frametitle{CGLS: Conjugate Gradient Least-Squares}
\begin{verbatim}
function x = mycgls(A,c,k,tol)
n = size(A,2);
x = zeros(n,1);
d = A'*c;   r = c;
normr2 = d'*d;
for j=1:k
    Ad = A*d; alpha = normr2/(Ad'*Ad); 
    x =x+alpha*d;
    r = r - alpha*Ad;
    s = A'*r;
    normr2New = d'*d;
    if normr2New<tol;return; end
    beta = normr2New/normr2;
    normr2 = normr2New;
    d = s + beta*d;
end
\end{verbatim}



\end{frame}

\begin{frame}
	\frametitle{Conjugate Gradient Least-Squares}
\begin{columns}
	\column{.7\textwidth}
	\begin{itemize}
	\item
	Uses the structure of the problem to obtain stable implementation
	\item
	Typically converges much faster than SD
	\item 
	Accelerate using post conditioning
	$$ \min_\bfx \hf \|\bfA \bfS \bfS^{-1} \bfx - \bfc\|^2$$
	\item 
	Faster convergence when eigenvalues of $\bfS^{\top} \bfA^{\top} \bfA \bfS$ are clustered.
	\end{itemize}
	
	\column{.3 \textwidth}
		
	\includegraphics[width=38mm]{exConjugateGradient2D}
\end{columns}
\end{frame}

\begin{frame}
	\frametitle{Iterative Regularization}

Consider
$$
	\min_\bfx \| \bfA \bfx - \bfb \|^2
$$
\begin{itemize}
\item
Assume that $\bfA$ has non-trivial null space
\item The matrix $\bfA^{\top}\bfA$ is not invertible
\item Can we still use iterative methods (CG, CGLS, \ldots)?
\end{itemize}

\bigskip

What are the properties of the iterates? 

\bigskip

Excellent introduction to computational inverse problems~\cite{Hansen1998,Vogel2002,Hansen2010}

\end{frame}




\begin{frame}
	\frametitle{Iterative Regularization: L-Curve}


The CGLS algorithm has the following properties
\begin{itemize}
\item For each iteration $\| \bfA \bfx_k -\bfc \|^2 \le \| \bfA \bfx_{k-1} -\bfc \|^2$
\item If starting from  $\bfx=0$ then $\|\bfx_k  \|^2 \ge \|  \bfx_{k-1}  \|^2$

\item $\bfx_1,\bfx_2,\ldots$ converges to the minimum norm solution of the problem
\item Plotting $\|\bfx_k\|^2$ vs $\| \bfA\bfx_k -\bfc \|^2$ typically has the shape of an L-curve
\end{itemize}



\begin{center}
\includegraphics[width=6cm]{lcurve.jpg}
\end{center}
\end{frame}


\begin{frame}
	\frametitle{Cross Validation - 1}

Finding good least-squares solution requires good parameter selection.
\begin{itemize}
\item $\lambda$ when using Tikhonov regularization (weight decay)
\item number of iteration (for SD and CGLS)
\end{itemize}

\bigskip

Suppose that we have two different ``solutions''

$$\bfx_1\quad  \rightarrow \quad  \|\bfx_1\|^2 = \eta_1 \quad \|\bfA \bfx_1 - \bfc\|^2 = \rho_1. $$
$$\bfx_2\quad  \rightarrow \quad  \|\bfx_2\|^2 = \eta_2 \quad \|\bfA \bfx_2 - \bfc\|^2 = \rho_2. $$

How to decide which one is better?

\end{frame}

\begin{frame}
	\frametitle{Cross Validation - 2}

Goal: Gauge how well the model can predict new examples.

\bigskip

Let $\{ \bfA_{\rm CV}, \bfc_{\rm CV} \}$ be data that is {\bf not used} for the training 

\bigskip

Idea: If $\|\bfA_{\rm CV} \bfx_1 - \bfc_{\rm CV}\|^2 \le 
\|\bfA_{\rm CV} \bfx_2 - \bfc_{\rm CV}\|^2$,
then $\bfx_1$ is a better solution that $\bfx_2$.

\bigskip

\pause

When the solution depends on some hyper-parameter(s) $\lambda$, we can phrase this as bi-level optimization problem
\begin{equation*}
 \lambda^* = \argmin_{\lambda} \|\bfA_{\rm CV} \bfx(\lambda) - \bfc_{\rm CV}\|^2,
\end{equation*}
where  $\bfx(\lambda) = \argmin_\bfx \hf\|\bfA \bfx - \bfx \|^2 + \frac{\lambda}{2} \|\bfx\|^2$.

\end{frame}

\begin{frame}
	\frametitle{Cross Validation - 3}

To assess the final quality of the solution cross validation is not sufficient (why?).

\bigskip

Need a final testing set.

\bigskip

Procedure
\begin{itemize}
\item Divide the data into 3 groups $\{ \bfA_{\rm train}, \bfA_{\rm CV}, \bfA_{\rm test} \}$.
\item Use $\bfA_{\rm train}$ to estimate $\bfx(\lambda)$
\item Use $\bfA_{\rm CV}$ to estimate $\lambda$
\item Use $\bfA_{\rm test}$ to assess the quality of the solution
\end{itemize}

\pause

{\bf Important} -  we are not allowed to use $\bfA_{\rm test}$ to tune parameters!

\end{frame}

%
% \section{Coding} % (fold)
% \label{sec:coding}


% \begin{frame}[fragile]
% 	\frametitle{Project: Iterative Methods for Regression}
%
% 	For the MNIST dataset (CIFAR-10 optional)
% 	\begin{itemize}
% 		\item solve the problem with
% 		\item write a steepest descent for least-squares problems
% 		\begin{verbatim}
% 			function x = sdLeastSquares(A,c,x0,maxIter)
% 		\end{verbatim}
% 		\item write a conjugate gradient code
% 		\begin{verbatim}
% 			function x = cgLeastSquares(A,c,maxIter)
% 		\end{verbatim}
% 	\end{itemize}
% \end{frame}

\begin{frame}
	\frametitle{$\Sigma$ : Linear Least-Squares}
		\begin{equation*}
			 \min_{\bfW} \hf \|\bfW \bfY - \bfC\|^2_F \quad \leadsto \quad\min_{\bfx} \hf \|\bfA \bfx - \bfb\|^2_F
		\end{equation*}
	\begin{itemize}
		\item optimality conditions: normal equations
		\item typically requires regularization
		\item direct: Tikhonov weight decay $\leadsto$ need to choose $\lambda$
		\item iterative: steepest descent, CG $\leadsto$ need to choose maximum number of iterations.
		\item classification: how to interpret output $\bfW \bfY$ (not a probability!)
		\item image inpainting: effective choice of regularizer and parameter.
	\end{itemize}
\end{frame}

\begin{frame}[allowframebreaks]
	\frametitle{References}
\bibliographystyle{abbrv}
\bibliography{NumDNN}

\end{frame}
\end{document}