20120110-ICERM.tex

% \documentclass[handout]{beamer}
\documentclass{beamer}

\mode<presentation>
{
  \usetheme{default}
  \usefonttheme[onlymath]{serif}
  % \usetheme{Singapore}
  % \usetheme{Warsaw}
  % \usetheme{Malmoe}
  % \useinnertheme{circles}
  % \useoutertheme{infolines}
  % \useinnertheme{rounded}

  \setbeamercovered{transparent=5}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{textpos,alltt,listings,multirow,ulem,siunitx}
\usepackage{pdfpages}
\newcommand\hmmax{0}
\newcommand\bmmax{0}
\usepackage{bm}

% font definitions, try \usepackage{ae} instead of the following
% three lines if you don't like this look
\usepackage{mathptmx}
\usepackage[scaled=.90]{helvet}
% \usepackage{courier}
\usepackage[T1]{fontenc}
\usepackage{tikz}
\usetikzlibrary[shapes,shapes.arrows,arrows,shapes.misc,fit,positioning]

% \usepackage{pgfpages}
% \pgfpagesuselayout{4 on 1}[a4paper,landscape,border shrink=5mm]

\usepackage{JedMacros}

\title{Toward less synchronous composable multilevel methods for implicit multiphysics simulation}
\author{Jed Brown\inst{1}, Mark Adams\inst{2}, Peter Brune\inst{1}, Matt Knepley\inst{3}, Barry Smith\inst{1}}


% - Use the \inst command only if there are several affiliations.
% - Keep it simple, no one is interested in your street address.
\institute
{
  \inst{1}{Mathematics and Computer Science Division, Argonne National Laboratory} \\
  \inst{2}{Columbia University} \\
  \inst{3}{Computation Institute, University of Chicago}
}

\date{2012-01-10}

% This is only inserted into the PDF information catalog. Can be left
% out.
\subject{Talks}


% If you have a file called "university-logo-filename.xxx", where xxx
% is a graphic format that can be processed by latex or pdflatex,
% resp., then you can add a logo as follows:

% \pgfdeclareimage[height=0.5cm]{university-logo}{university-logo-filename}
% \logo{\pgfuseimage{university-logo}}



% Delete this, if you do not want the table of contents to pop up at
% the beginning of each subsection:
% \AtBeginSubsection[]
% {
% \begin{frame}<beamer>
%   \frametitle{Outline}
%   \tableofcontents[currentsection,currentsubsection]
% \end{frame}
% }

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

% If you wish to uncover everything in a step-wise fashion, uncomment
% the following command:

% \beamerdefaultoverlayspecification{<+->}

\begin{document}
\lstset{language=C}
\normalem

\begin{frame}
  \titlepage
\end{frame}

\section{Multiphysics and methods}
\input{slides/MultiphysicsExamples.tex}
\input{slides/MonolithicOrSplit.tex}
\input{slides/FieldSplit.tex}

\section{Coupling software in PETSc}
\input{slides/PETSc/Coupling.tex}
\input{slides/PETSc/LocalSpaces.tex}
\input{slides/PETSc/MatGetLocalSubMatrix.tex}
\setbeamertemplate{background canvas}{}
\includepdf[pages=1]{davemay.pdf}

\section{Hardware and consequences}
\begin{frame}{On-node hardware roadmap}
  \begin{block}{Hardware trends}
    \begin{itemize}
    \item More cores (keep hearing $\bigO(1000)$ per node)
    \item Long vector registers (32B for AVX and BG/Q, 64B for MIC)
    \item Must use SMT to hide memory latency
    \item Must use SMT for floating point performance (GPU, BG/Q)
    \item Large penalty for non-contiguous memory access
    \end{itemize}
  \end{block}
  \begin{block}{``Free flops'', but how can we use them?}
    \begin{itemize}
    \item High order methods good: better accuracy per storage
    \item High order methods bad: work unit gets larger
    \item GPU threads have very little memory, must keep work unit small
    \item Need library composability, keep user contribution embarrassingly parallel
    \end{itemize}
  \end{block}
\end{frame}

\begin{frame}{How to program this beast?}
  \begin{itemize}
  \item Decouple physics from discretization
    \begin{itemize*}
    \item Expose small, embarrassingly parallel operations to user
    \item Library schedules user threads for reuse between kernels
    \item User provides physics in kernels run at each quadrature point
    \item Continuous weak form: find $u \in \VV_D$
      \[ v^T F(u) \sim \int_\Omega v \cdot {\color{green!70!black} f_0(u,\nabla u)}
      + \nabla v \tcolon {\color{green!70!black} f_1(u,\nabla u)} = 0, \qquad \forall v \in \VV_0 \]
    \item Similar form at faces, but may involve Riemann solve
    \end{itemize*}
  \item Library manages reductions
    \begin{itemize*}
    \item Interpolation and differentiation on elements
    \item Exploit tensor product structure to keep working set small
    \item Assembly into solution/residual vector (sum over elements)
    \end{itemize*}
  \end{itemize}
\end{frame}

\input{slides/Dohp/TensorFEM.tex}
\input{slides/Dohp/RepresentationOfJacobians.tex}
\newcommand\smallterm[1]{{\color{gray} #1}}
\begin{frame}{Conservative (non-Boussinesq) two-phase ice flow}
  Find momentum density $\rho\uu$, pressure $p$, and total energy density $E$:
  \begin{gather*}
    (\rho\uu)_t + \div (\smallterm{\rho\uu\otimes\uu} - \eta D\uu_i + p\bm 1) - \rho \bm g = 0 \\
    \rho_t + \div \rho\uu = 0 \\
    E_t + \div \big((E+p)\uu - k_T\nabla T - k_\omega\nabla\omega \big) - \eta D\uu_i\tcolon D\uu_i - \smallterm{\rho\uu\cdot\bm g} = 0
  \end{gather*}
\begin{itemize}
\item Solve for density $\rho$, ice velocity $\uu_i$, temperature $T$, and melt fraction $\omega$ using constitutive relations.
  \begin{itemize}
  \item Simplified constitutive relations can be solved explicitly.
  \item Temperature, moisture, and strain-rate dependent rheology $\eta$.
  \item High order FEM, typically $Q_3$ momentum \& energy
  \end{itemize}
\item DAEs solved implicitly after semidiscretizing in space.
\item Preconditioning using nested fieldsplit
\end{itemize}
\end{frame}
\input{slides/VHTSolvers.tex}

\input{slides/Dohp/TensorVsAssembly.tex}
\input{slides/HardwareArithmeticIntensity.tex}

\begin{frame}{Prospects for reducing synchronization}
  \begin{itemize}
  \item Dot products and norms
    \begin{itemize}
    \item orthogonality is a powerful concept
    \item dot product/norm fusion in CG variants
    \item latency-tolerant Krylov methods, TSQR for GMRES
    \item nonlinear methods (e.g. NGMRES, BFGS, line searches)
    \item hierarchical methods to limit system-wide norms
    \item setting up smoothers and coarsening rates for AMG
    \end{itemize}
  \item additive coarse grids
  \item subphysics on subcommunicators, even within multigrid context
  \item $s$-step methods (and other fusion)
    \begin{itemize}
    \item usually spoiled by algorithmic requirements of preconditioning
    \item relevant for multigrid smoothers
    \item difficult crossovers for 3D problems
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{$S$-step methods in 3D}
  \includegraphics[width=1.1\textwidth]{figures/SStep.pdf}
\end{frame}

\begin{frame}{Multigrid is \emph{always} strong scaling}
  \begin{itemize}
  \item Finest level is chosen by the application (might have big subdomains)
  \item All coarsened levels choose communicator size based on strong scaling limit
  \item Optimizing the strong scaling limit pays off consistently
  \item Rapid coarsening is important (2:1 semi-coarsening not okay any more)
  \end{itemize}
\end{frame}

\begin{frame}{Software challenges}
  \begin{itemize}
  \item Which interfaces do users have to interact with?
    \begin{itemize}
    \item ``F''ramework vs library
    \item Extensibility is critical for multiphysics
    \end{itemize}
  \item Asynchronous interfaces crossing module boundaries
    \begin{itemize}
    \item How to ensure progress?
    \end{itemize}
  \item Merge communication on multiple levels or between multiple physics
  \item Fusing coarse level operations
  \item Working with non-nested communicators is tricky
  \item Current solutions for hierarchical memory are bad for libraries
    \begin{itemize}
    \item I want a communicator-like object
    \item I want a way to allocate memory explicitly/relative to algorithmic dependencies instead of implicit ``first touch''
    \end{itemize}
  \item Time integration: IMEX, multirate, parallel in time
  \end{itemize}
\end{frame}

\end{document}