diff --git a/src/diff.c b/src/diff.c
index df3dea0a179c7..eddf33165628d 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -119,7 +119,12 @@ diff_buf_delete(buf_T *buf)
 	    tp->tp_diffbuf[i] = NULL;
 	    tp->tp_diff_invalid = TRUE;
 	    if (tp == curtab)
-		diff_redraw(TRUE);
+	    {
+		// don't redraw right away, more might change or buffer state
+		// is invalid right now
+		need_diff_redraw = TRUE;
+		redraw_later(VALID);
+	    }
 	}
     }
 }
@@ -670,7 +675,8 @@ diff_redraw(
 
     need_diff_redraw = FALSE;
     FOR_ALL_WINDOWS(wp)
-	if (wp->w_p_diff)
+	// when closing windows or wiping buffers skip invalid window
+	if (wp->w_p_diff && buf_valid(wp->w_buffer))
 	{
 	    redraw_win_later(wp, SOME_VALID);
 	    if (wp != curwin)
diff --git a/src/testdir/test_diffmode.vim b/src/testdir/test_diffmode.vim
index 03c6c798e05e7..afa8f891be553 100644
--- a/src/testdir/test_diffmode.vim
+++ b/src/testdir/test_diffmode.vim
@@ -1603,5 +1603,17 @@ func Test_diff_scroll()
   call delete('Xright')
 endfunc
 
+" This was trying to update diffs for a buffer being closed
+func Test_diff_only()
+  silent! lfile
+  set diff
+  lopen
+  norm o
+  silent! norm o
+
+  set nodiff
+  %bwipe!
+endfunc
+
 
 " vim: shiftwidth=2 sts=2 expandtab
diff --git a/src/version.c b/src/version.c
index cf2e77d9fb791..c847d590527cc 100644
--- a/src/version.c
+++ b/src/version.c
@@ -735,6 +735,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    5163,
 /**/
     5162,
 /**/