diff --git a/Dockerfile b/Dockerfile
index 934ab11b..5814eeeb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,20 +4,19 @@ USER root
 
 RUN apt-get -y -qq update \
  && apt-get -y -qq install \
-        dbus-x11 \
-        xfce4 \
-        xfce4-panel \
-        xfce4-session \
-        xfce4-settings \
-        xorg \
-        xubuntu-icon-theme \
-        fonts-dejavu \
-    # Disable the automatic screenlock since the account password is unknown
- && apt-get -y -qq remove xfce4-screensaver \
+        ubuntu-mate-desktop \
+        vim \
+ && add-apt-repository -y ppa:mozillateam/ppa \
+ && printf 'Package: firefox*\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001\n' > /etc/apt/preferences.d/firefox \
+ && apt-get install -y -q firefox \
+ && apt-get purge -y -q \
+        blueman \
+        mate-screensaver \
+ && apt-get autoremove -y -q \
     # chown $HOME to workaround that the xorg installation creates a
     # /home/jovyan/.cache directory owned by root
     # Create /opt/install to ensure it's writable by pip
- && mkdir -p /opt/install \
+ && mkdir -p /opt/install $HOME/.vnc \
  && chown -R $NB_UID:$NB_GID $HOME /opt/install \
  && rm -rf /var/lib/apt/lists/*
 
@@ -28,7 +27,6 @@ RUN if [ "${vncserver}" = "tigervnc" ]; then \
         apt-get -y -qq update; \
         apt-get -y -qq install \
             tigervnc-standalone-server \
-            tigervnc-xorg-extension \
         ; \
         rm -rf /var/lib/apt/lists/*; \
     fi
@@ -57,3 +55,12 @@ COPY --chown=$NB_UID:$NB_GID . /opt/install
 RUN . /opt/conda/bin/activate && \
     pip install -e /opt/install && \
     jupyter server extension enable jupyter_remote_desktop_proxy
+
+COPY --chown=$NB_UID:$NB_GID start-mate.sh $HOME/.vnc/xstartup
+
+# Add some shortcuts to the desktop
+RUN mkdir -p $HOME/Desktop && \
+    ln -s \
+        /usr/share/applications/mate-terminal.desktop \
+        /usr/share/applications/firefox.desktop \
+        $HOME/Desktop
diff --git a/start-mate.sh b/start-mate.sh
new file mode 100755
index 00000000..93d80a6e
--- /dev/null
+++ b/start-mate.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Firefox sandboxing may not work in a container
+# https://wiki.mozilla.org/Security/Sandbox#Environment_variables
+export MOZ_DISABLE_CONTENT_SANDBOX=1
+export MOZ_DISABLE_GMP_SANDBOX=1
+export MOZ_DISABLE_RDD_SANDBOX=1
+export MOZ_DISABLE_SOCKET_PROCESS_SANDBOX=1
+
+exec dbus-launch mate-session "$@"