diff --git a/camel/providers/imap/camel-imap-folder.c b/camel/providers/imap/camel-imap-folder.c
index 657e547..1ee2271 100644
--- a/camel/providers/imap/camel-imap-folder.c
+++ b/camel/providers/imap/camel-imap-folder.c
@@ -1259,7 +1259,9 @@ imap_rescan (CamelFolder *folder,
 		 * Because server always reports NULL, which means none, which
 		 * will remove user's flags from local machine, which is bad.
 		*/
-		if ((folder->permanent_flags & CAMEL_MESSAGE_USER) != 0 && merge_custom_flags (info, new[j].custom_flags))
+		if ((folder->permanent_flags & CAMEL_MESSAGE_USER) != 0 &&
+		    (iinfo->info.flags & CAMEL_MESSAGE_FOLDER_FLAGGED) == 0 &&
+		    merge_custom_flags (info, new[j].custom_flags))
 			changed = TRUE;
 
 		if (changed) {
