diff --git a/src/vte.c b/src/vte.c
index c5f1af7..62ad067 100644
--- a/src/vte.c
+++ b/src/vte.c
@@ -6372,8 +6372,6 @@ vte_terminal_start_selection(VteTerminal *terminal, long x, long y,
 {
 	long delta;
 
-	terminal->pvt->selection_block_mode = !!(terminal->pvt->modifiers & GDK_CONTROL_MASK);
-
 	if (terminal->pvt->selection_block_mode)
 		selection_type = selection_type_char;
 
@@ -7178,6 +7176,9 @@ vte_terminal_button_press(GtkWidget *widget, GdkEventButton *event)
 				terminal->pvt->selecting = TRUE;
 				handled = TRUE;
 			}
+                        if (start_selecting || extend_selecting) {
+                                terminal->pvt->selection_block_mode = !!(terminal->pvt->modifiers & GDK_CONTROL_MASK);
+                        }
 			break;
 		/* Paste if the user pressed shift or we're not sending events
 		 * to the app. */
