diff --git a/lisp/whitespace.el b/lisp/whitespace.el
index 240f99effc2..41b0a34f9ea 100644
--- a/lisp/whitespace.el
+++ b/lisp/whitespace.el
@@ -2403,9 +2403,11 @@ to `indent-tabs-mode' and `tab-width'."
       ;; Remember whether a buffer has a local display table.
       (unless whitespace-display-table-was-local
 	(setq whitespace-display-table-was-local t)
-        (unless (or whitespace-mode global-whitespace-mode)
-	      (setq whitespace-display-table
-	      (copy-sequence buffer-display-table)))
+        ;; Save the old table so we can restore it when
+        ;; `whitespace-mode' is switched off again.
+        (when (or whitespace-mode global-whitespace-mode)
+	  (setq whitespace-display-table
+	        (copy-sequence buffer-display-table)))
 	;; Assure `buffer-display-table' is unique
 	;; when two or more windows are visible.
 	(setq buffer-display-table