diff --git a/src/vte-private.h b/src/vte-private.h
index 7611e05..cad2894 100644
--- a/src/vte-private.h
+++ b/src/vte-private.h
@@ -358,6 +358,7 @@ struct _VteTerminalPrivate {
 	int mouse_tracking_mode; /* this is of type MouseTrackingMode,
 				    but we need to guarantee its type. */
         guint mouse_pressed_buttons;      /* bits 0, 1, 2 resp. for buttons 1, 2, 3 */
+        gboolean mouse_button1_highlights;
         gboolean mouse_button2_pasted;
 	long mouse_last_x, mouse_last_y;
 	gboolean mouse_autohide;
diff --git a/src/vte.c b/src/vte.c
index 23449a5..6866b59 100644
--- a/src/vte.c
+++ b/src/vte.c
@@ -7061,10 +7061,7 @@ vte_terminal_motion_notify(GtkWidget *widget, GdkEventMotion *event)
 						     selection_type_char);
 		}
 
-		if (terminal->pvt->selecting &&
-		    ((terminal->pvt->modifiers & GDK_SHIFT_MASK) ||
-		      !terminal->pvt->mouse_tracking_mode))
-		{
+		if (terminal->pvt->selecting && terminal->pvt->mouse_button1_highlights) {
 			_vte_debug_print(VTE_DEBUG_EVENTS, "Mousing drag 1.\n");
 			vte_terminal_extend_selection(terminal,
 						      x, y, FALSE, FALSE);
@@ -7149,6 +7146,7 @@ vte_terminal_button_press(GtkWidget *widget, GdkEventButton *event)
 			if (terminal->pvt->mouse_tracking_mode) {
 				if (terminal->pvt->modifiers & GDK_SHIFT_MASK) {
 					start_selecting = TRUE;
+                                        terminal->pvt->mouse_button1_highlights = TRUE;
 				}
 			} else {
 				/* If the user hit shift, then extend the
@@ -7164,6 +7162,7 @@ vte_terminal_button_press(GtkWidget *widget, GdkEventButton *event)
 				} else {
 					start_selecting = TRUE;
 				}
+                                terminal->pvt->mouse_button1_highlights = TRUE;
 			}
 			if (start_selecting) {
 				vte_terminal_deselect_all(terminal);
@@ -7219,8 +7218,7 @@ vte_terminal_button_press(GtkWidget *widget, GdkEventButton *event)
 							     selection_type_char);
 				handled = TRUE;
 			}
-			if ((terminal->pvt->modifiers & GDK_SHIFT_MASK) ||
-			    !terminal->pvt->mouse_tracking_mode) {
+                        if (terminal->pvt->mouse_button1_highlights) {
 				vte_terminal_start_selection(terminal,
 							     x, y,
 							     selection_type_word);
@@ -7240,8 +7238,7 @@ vte_terminal_button_press(GtkWidget *widget, GdkEventButton *event)
 				x, y + (terminal->pvt->char_height * delta));
 		switch (event->button) {
 		case 1:
-			if ((terminal->pvt->modifiers & GDK_SHIFT_MASK) ||
-			    !terminal->pvt->mouse_tracking_mode) {
+                        if (terminal->pvt->mouse_button1_highlights) {
 				vte_terminal_start_selection(terminal,
 							     x, y,
 							     selection_type_line);
@@ -7294,7 +7291,8 @@ vte_terminal_button_release(GtkWidget *widget, GdkEventButton *event)
 				event->button, x, y);
 		switch (event->button) {
 		case 1:
-			handled = _vte_terminal_maybe_end_selection (terminal);
+                        if (terminal->pvt->mouse_button1_highlights)
+                                handled = _vte_terminal_maybe_end_selection (terminal);
 			break;
 		case 2:
                         handled = terminal->pvt->mouse_button2_pasted;
@@ -7375,6 +7373,7 @@ vte_terminal_focus_out(GtkWidget *widget, GdkEventFocus *event)
 		/* Mark the cursor as invisible to disable hilite updating */
 		terminal->pvt->mouse_cursor_visible = FALSE;
                 terminal->pvt->mouse_pressed_buttons = 0;
+                terminal->pvt->mouse_button1_highlights = FALSE;
                 terminal->pvt->mouse_button2_pasted = FALSE;
 	}
 
@@ -11970,6 +11969,7 @@ vte_terminal_reset(VteTerminal *terminal,
 	/* Reset mouse motion events. */
 	pvt->mouse_tracking_mode = MOUSE_TRACKING_NONE;
         pvt->mouse_pressed_buttons = 0;
+        pvt->mouse_button1_highlights = FALSE;
         pvt->mouse_button2_pasted = FALSE;
 	pvt->mouse_last_x = 0;
 	pvt->mouse_last_y = 0;
