diff --git a/src/frame.c b/src/frame.c
index cf894db6836..7f4bf274ad9 100644
--- a/src/frame.c
+++ b/src/frame.c
@@ -4838,6 +4838,7 @@ gui_set_font (struct frame *f, Lisp_Object arg, Lisp_Object oldval)
 	     new image cache with IWIDTH.  */
 	  FRAME_IMAGE_CACHE (f)->refcount--;
 	  FRAME_IMAGE_CACHE (f) = share_image_cache (f);
+	  FRAME_IMAGE_CACHE (f)->refcount++;
 	}
     }