diff --git a/lisp/auth-source.el b/lisp/auth-source.el
index 6cdea11c1ee..bc660b2f4ab 100644
--- a/lisp/auth-source.el
+++ b/lisp/auth-source.el
@@ -2602,14 +2602,14 @@ point is moved into the passwords (see `authinfo-hide-elements').
 (defvar read-passwd--mode-line-icon nil
   "Propertized mode line icon for showing/hiding passwords.")
 
-(defvar read-passwd--hide-password t
-  "Toggle whether password should be hidden in minibuffer.")
+(defvar read-passwd--password-hidden nil
+  "Flag indicating whether password in minibuffer is hidden.")
 
 (defun read-passwd--hide-password ()
   "Make password in minibuffer hidden or visible."
   (let ((beg (minibuffer-prompt-end)))
     (dotimes (i (1+ (- (buffer-size) beg)))
-      (if read-passwd--hide-password
+      (if read-passwd--password-hidden
           (put-text-property
            (+ i beg) (+ 1 i beg) 'display (string (or read-hide-char ?*)))
         (remove-list-of-text-properties (+ i beg) (+ 1 i beg) '(display)))
@@ -2617,9 +2617,10 @@ point is moved into the passwords (see `authinfo-hide-elements').
        (+ i beg) (+ 1 i beg)
        'help-echo "C-u: Clear password\nTAB: Toggle password visibility"))))
 
-(defun read-passwd-toggle-visibility ()
+(defun read-passwd-toggle-visibility (&optional force)
   "Toggle minibuffer contents visibility.
-Adapt also mode line."
+Adapt also mode line.  If optional FORCE is non-nil, hide the minibuffer
+contents."
   (interactive)
   (let ((win (active-minibuffer-window)))
     (unless win (error "No active minibuffer"))
@@ -2627,12 +2628,13 @@ Adapt also mode line."
     ;; mini-buffer.
     (with-current-buffer (window-buffer win)
       (when (memq 'read-passwd-mode local-minor-modes)
-        (setq read-passwd--hide-password (not read-passwd--hide-password))
+        (setq read-passwd--password-hidden
+              (or force (not read-passwd--password-hidden)))
         (setq read-passwd--mode-line-icon
               `(:propertize
                 ,(if icon-preference
                      (icon-string
-                      (if read-passwd--hide-password
+                      (if read-passwd--password-hidden
                           'read-passwd--show-password-icon
                         'read-passwd--hide-password-icon))
                    "")
@@ -2652,6 +2654,9 @@ Adapt also mode line."
   "C-u" #'delete-minibuffer-contents ;bug#12570
   "TAB" #'read-passwd-toggle-visibility)
 
+(defvar read-passwd--mini-buffers nil
+  "List of minibuffers where `read-passwd' is active.")
+
 (define-minor-mode read-passwd-mode
   "Toggle visibility of password in minibuffer."
   :group 'mode-line
@@ -2659,21 +2664,25 @@ Adapt also mode line."
   :keymap read-passwd-map
   :version "30.1"
 
-  (setq read-passwd--hide-password nil)
-  (or global-mode-string (setq global-mode-string '("")))
-
-  (let ((mode-string '(:eval read-passwd--mode-line-icon)))
-    (if read-passwd-mode
-        ;; Add `read-passwd--mode-line-icon'.
-        (or (member mode-string global-mode-string)
-            (setq global-mode-string
-	          (append global-mode-string (list mode-string))))
-      ;; Remove `read-passwd--mode-line-icon'.
-      (setq global-mode-string
-	    (delete mode-string global-mode-string))))
-
+  (unless read-passwd-mode
+    (setq read-passwd--mini-buffers
+          (delq (current-buffer) read-passwd--mini-buffers)))
+  (unless read-passwd--mini-buffers
+    (let ((mode-string '(:eval read-passwd--mode-line-icon)))
+      (if read-passwd-mode
+          ;; Add `read-passwd--mode-line-icon'.
+          (or (member mode-string global-mode-string)
+              (setq global-mode-string
+	            (append global-mode-string (list mode-string))))
+        ;; Remove `read-passwd--mode-line-icon'.
+        (setq global-mode-string
+	      (delete mode-string global-mode-string)))))
   (when read-passwd-mode
-    (read-passwd-toggle-visibility)))
+    (push (current-buffer) read-passwd--mini-buffers))
+  ;; Always hide the current password.
+  (when read-passwd--mini-buffers
+    (with-current-buffer (car read-passwd--mini-buffers)
+      (read-passwd-toggle-visibility t))))
 
 (defvar overriding-text-conversion-style)