20120427-NREL.tex

% \documentclass[handout]{beamer}
\documentclass{beamer}

\mode<presentation>
{
  \usetheme{default}
  \usefonttheme[onlymath]{serif}
  % \usetheme{Singapore}
  % \usetheme{Warsaw}
  % \usetheme{Malmoe}
  % \useinnertheme{circles}
  % \useoutertheme{infolines}
  % \useinnertheme{rounded}

  \setbeamercovered{transparent=100}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{textpos,alltt,listings,multirow,ulem,siunitx}
\usepackage{pdfpages}
\usepackage{multimedia}
\newcommand\hmmax{0}
\newcommand\bmmax{0}
\usepackage{bm}

% font definitions, try \usepackage{ae} instead of the following
% three lines if you don't like this look
\usepackage{mathptmx}
\usepackage[scaled=.90]{helvet}
% \usepackage{courier}
\usepackage[T1]{fontenc}
\usepackage{tikz}
\usetikzlibrary[shapes,shapes.arrows,arrows,shapes.misc,fit,positioning,trees,mindmap,backgrounds]

% \usepackage{pgfpages}
% \pgfpagesuselayout{4 on 1}[a4paper,landscape,border shrink=5mm]

\usepackage{JedMacros}

\title{Towards High Throughput Composable Multilevel Solvers for Implicit Multiphysics Simulation}
\subtitle{This talk: \url{http://59A2.org/files/20120427-NREL.pdf}}
\author{Jed Brown\inst{1},\\
Peter Brune\inst{1}, Matt Knepley\inst{2}, Dave May\inst{3}, Barry Smith\inst{1}}


% - Use the \inst command only if there are several affiliations.
% - Keep it simple, no one is interested in your street address.
\institute
{
  \inst{1}{Mathematics and Computer Science Division, Argonne National Laboratory} \\
  \inst{2}{Computation Institute, University of Chicago} \\
  \inst{3}{ETH Z\"urich}
}

\date{NREL 2012-04-27}

% This is only inserted into the PDF information catalog. Can be left
% out.
\subject{Talks}


% If you have a file called "university-logo-filename.xxx", where xxx
% is a graphic format that can be processed by latex or pdflatex,
% resp., then you can add a logo as follows:

% \pgfdeclareimage[height=0.5cm]{university-logo}{university-logo-filename}
% \logo{\pgfuseimage{university-logo}}



% Delete this, if you do not want the table of contents to pop up at
% the beginning of each subsection:
% \AtBeginSubsection[]
% {
% \begin{frame}<beamer>
%   \frametitle{Outline}
%   \tableofcontents[currentsection,currentsubsection]
% \end{frame}
% }

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

% If you wish to uncover everything in a step-wise fashion, uncomment
% the following command:

% \beamerdefaultoverlayspecification{<+->}

\begin{document}
\lstset{language=C}
\normalem

\begin{frame}
  \titlepage
\end{frame}

\input{slides/CSEMindmap.tex}

\section{Composable Solvers}
\input{slides/MultiphysicsExamples.tex}
\input{slides/MonolithicOrSplit.tex}
\input{slides/PETSc/Coupling.tex}
\input{slides/FieldSplit.tex}
\input{slides/PETSc/LocalSpaces.tex}
\input{slides/SNES/MultiphysicsAssemblyJacobians.tex}
\input{slides/Stokes/OptionsTour.tex}
\input{slides/Stokes/CoupledMGOptions.tex}
\input{slides/SNES/MonolithicFASMultistage.tex}
\input{slides/SNES/NonlinearSolversList.tex}
\input{slides/SNES/QuasiNewtonRevisited.tex}

\begin{frame}{Blocking communication is the enemy}
  \includegraphics{figures/PipelinedGMRES} \\
  In PETSc, Block Jacobi with ILU preconditioning, {\tt -ksp\_type pgmres}
\end{frame}
\begin{frame}{GMRES communication (16 procs)}
  \includegraphics[width=\textwidth]{figures/GMRESJumpshot}
\end{frame}
\begin{frame}{GMRES communication (32 procs)}
  \includegraphics[width=\textwidth]{figures/PGMRESJumpshot}
\end{frame}
\begin{frame}{GMRES communication (32 procs)}
  \includegraphics[width=\textwidth]{figures/GMRESJumpshotN32}
\end{frame}
\begin{frame}{Pipelined GMRES communication (32 procs)}
  \includegraphics[width=\textwidth]{figures/PGMRESJumpshotN32}
\end{frame}

\input{slides/DrunkenSeaman.tex}
\setbeamertemplate{background canvas}{}
\includepdf[pages=1-3]{davemay.pdf}

\input{slides/GroundingLine/ThwaitesPIG.tex}
\input{slides/GroundingLine/Steepness.tex}
\begin{frame}
  \includegraphics[width=0.8\textwidth]{figures/VHT/VHTJakoContourStream}
\end{frame}
\input{slides/VHTMotivation.tex}
\input{slides/VHTEquationsSimple.tex}
\input{slides/VHTSolvers.tex}
%\input{slides/VHTSolverOptions.tex}

\input{slides/AllenCahnOptions.tex}

\section{Time Integration}
\input{slides/IMEXMotivation.tex}
\input{slides/PETSc/TSARKIMEX.tex}
\begin{frame}{Stiff advection-reaction accuracy}
  \includepdf[scale=0.8]{figures/TS/ConvTest_PETSc_case_1002_Norm_Inf_6k_1-000_800.pdf}
\end{frame}
\begin{frame}{3D Resistive MHD}
  \includegraphics[width=\textwidth]{figures/MHD.png}
  \begin{itemize}
  \item Designing IMEX methods with specific stability properties, e.g. $L$-stable implicit, $A$-stable embedded, SSP explicit with optimal stability on the imaginary axis.
  \end{itemize}
\end{frame}

\begin{frame}{Outlook on Solver Composition}
  \begin{itemize}
  \item Unintrusive composition of multigrid and block preconditioning
  \item We can build many preconditioners from the literature \\
    \emph{on the command line}
  \item User code does not depend on matrix format, preconditioning method, nonlinear solution method, time integration method (implicit or IMEX), or size of coupled system (except for driver).
  \end{itemize}
  \begin{block}{In development}
    \begin{itemize}
    \item Distributive relaxation, Vanka smoothers
    \item Algebraic coarsening of ``dual'' variables
    \item Improving operator-dependent semi-geometric multigrid
    \item More automatic spectral analysis and smoother optimization
    \item Automated support for mixing analysis into levels
  \end{itemize}
\end{block}
\end{frame}

\section{Implementation Efficiency}
\input{slides/HardwareRoadmap.tex}

\begin{frame}{How to program this beast?}
  \begin{itemize}
  \item Decouple physics from discretization
    \begin{itemize*}
    \item Expose small, embarrassingly parallel operations to user
    \item Library schedules user threads for reuse between kernels
    \item User provides physics in kernels run at each quadrature point
    \item Continuous weak form: find $u \in \VV_D$
      \[ v^T F(u) \sim \int_\Omega v \cdot {\color{green!70!black} f_0(u,\nabla u)}
      + \nabla v \tcolon {\color{green!70!black} f_1(u,\nabla u)} = 0, \qquad \forall v \in \VV_0 \]
    \item Similar form at faces, but may involve Riemann solve
    \end{itemize*}
  \item Library manages reductions
    \begin{itemize*}
    \item Interpolation and differentiation on elements
    \item Interaction with neighbors (limiting, edge stabilization)
    \item Exploit tensor product structure to keep working set small
    \item Assembly into solution/residual vector (sum over elements)
    \end{itemize*}
  \end{itemize}
\end{frame}

\input{slides/Dohp/TensorFEM.tex}
\input{slides/Dohp/RepresentationOfJacobians.tex}
%\input{slides/Dohp/CPUTraversalCode.tex}
\input{slides/Dohp/TensorVsAssembly.tex}
\input{slides/HardwareArithmeticIntensity.tex}
\input{slides/GPU/FinerGrainedParallelismForFEM.tex}
\input{slides/GPU/FEMIntegrationTranspose.tex}
\begin{frame}{On preconditioning and multigrid}
  \begin{itemize}
  \item Often using assembled matrices for preconditioning
  \item Prefer matrix-free preconditioners for high hardware utilization
  \item Geometric $h$- and $p$-multigrid, could be FAS
  \item Smoothers build/solve with small dense matrices
    \begin{itemize}
    \item ``point'' matrices: can use single threads
    \item ``element'' matrices: need to cooperate within thread blocks
    \item I want a dense linear algebra library to be called collectively within a thread block
    \end{itemize}
  \item Multiplicative (Gauss-Seidel) is algorithmically nice
  \item Spectral analysis for polynomial/multi-stage smoothers
  \item Coarser levels better to do on CPU
    \begin{itemize}
    \item Potential for additive correction to run concurrently
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Outlook}
  \begin{itemize}
  \item Sparse matrix assembly (for preconditioning)
    \begin{itemize}
    \item $> 100$ GF/s for lowest order Stokes (Matt Knepley)
    \item common ``pointwise'' physics code with CPU implementation
    \item Dohp CPU version faster than libMesh and Deal.II for $Q_1$
    \item $Q_1$ assembly embedded in higher order is 8\% slower than hand-rolled
    \end{itemize}
  \item Matrix-free tensor-product versions reliably get about 70\% of peak flops
  \item Finer grained parallelism in GPU tensor product kernels
  \item Can't wait for OpenCL to implement indirect function calls
  \item Symbolic differentiation too slow, tired of hand-differentiation
    \begin{itemize}
    \item I want source-transformation AD with indirect function calls
    \end{itemize}
  \item Find correct amount of reuse between face and cell integration
  \item Riemann solves harder to vectorize
  \item Hide dispatch to pointwise kernels inside library
    \begin{itemize}
    \item Easy, but scary. Library/framework becomes \alert{\bf F}ramework.
    \item Interoperbility of user-rolled, library-provided, and generated traversal code.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}\LARGE
  \begin{itemize}
  \item Maximize science per Watt
  \item Huge scope remains at problem formulation
  \item Raise level of abstraction at which a problem is formally specified
  \item Algorithmic optimality is crucial
  \end{itemize}
\end{frame}

\end{document}