diff --git a/gnome-panel/menu.c b/gnome-panel/menu.c
index 6cb104b..e9249e7 100644
--- a/gnome-panel/menu.c
+++ b/gnome-panel/menu.c
@@ -240,8 +240,7 @@ icon_to_load_free (IconToLoad *icon)
 	if (!icon)
 		return;
 
-	if (icon->pixmap)
-		g_object_unref (icon->pixmap);
+	/* we don't own a ref to the GtkImage, so don't unref it */
 	icon->pixmap = NULL;
 
 	if (icon->gicon)
@@ -1456,8 +1455,10 @@ panel_load_menu_image_deferred (GtkWidget   *image_menu_item,
 	image = gtk_image_new ();
 	gtk_widget_set_size_request (image, icon_height, icon_height);
 
-	/* this takes over the floating ref */
-	icon->pixmap = g_object_ref_sink (G_OBJECT (image));
+	/* Do not make the structure own a ref to GtkImage: the structure
+	 * doesn't survive the GtkImage anyway (see g_signal_connect_data()
+	 * below) */
+	icon->pixmap = image;
 
 	icon->stock_id       = stock_id;
 	if (gicon)
@@ -1472,7 +1473,7 @@ panel_load_menu_image_deferred (GtkWidget   *image_menu_item,
 
 	g_object_set_data_full (G_OBJECT (image_menu_item),
 				"Panel:Image",
-				g_object_ref (image),
+				g_object_ref_sink (image),
 				(GDestroyNotify) g_object_unref);
 
 	gtk_image_menu_item_set_image (GTK_IMAGE_MENU_ITEM (image_menu_item),
