diff --git a/src/vte-private.h b/src/vte-private.h
index e05216f..c5c1fcd 100644
--- a/src/vte-private.h
+++ b/src/vte-private.h
@@ -358,6 +358,7 @@ struct _VteTerminalPrivate {
 	int mouse_tracking_mode; /* this is of type MouseTrackingMode,
 				    but we need to guarantee its type. */
         guint mouse_pressed_buttons;      /* bits 0, 1, 2 resp. for buttons 1, 2, 3 */
+        guint mouse_handled_buttons;      /* similar bitmap for buttons we handled ourselves */
 	long mouse_last_x, mouse_last_y;
 	gboolean mouse_autohide;
 	guint mouse_autoscroll_tag;
diff --git a/src/vte.c b/src/vte.c
index 805d8d2..29cd165 100644
--- a/src/vte.c
+++ b/src/vte.c
@@ -7200,6 +7200,12 @@ vte_terminal_button_press(GtkWidget *widget, GdkEventButton *event)
 		default:
 			break;
 		}
+                if (event->button >= 1 && event->button <= 3) {
+                        if (handled)
+                                terminal->pvt->mouse_handled_buttons |= (1 << (event->button - 1));
+                        else
+                                terminal->pvt->mouse_handled_buttons &= ~(1 << (event->button - 1));
+                }
 		/* If we haven't done anything yet, try sending the mouse
 		 * event to the app. */
 		if (handled == FALSE) {
@@ -7297,7 +7303,8 @@ vte_terminal_button_release(GtkWidget *widget, GdkEventButton *event)
 			handled = _vte_terminal_maybe_end_selection (terminal);
 			break;
 		case 2:
-			handled = TRUE;
+                        handled = (terminal->pvt->mouse_handled_buttons & 2) != 0;
+                        terminal->pvt->mouse_handled_buttons &= ~2;
 			break;
 		case 3:
 		default:
@@ -7374,6 +7381,7 @@ vte_terminal_focus_out(GtkWidget *widget, GdkEventFocus *event)
 		/* Mark the cursor as invisible to disable hilite updating */
 		terminal->pvt->mouse_cursor_visible = FALSE;
                 terminal->pvt->mouse_pressed_buttons = 0;
+                terminal->pvt->mouse_handled_buttons = 0;
 	}
 
 	terminal->pvt->has_focus = FALSE;
@@ -11968,6 +11976,7 @@ vte_terminal_reset(VteTerminal *terminal,
 	/* Reset mouse motion events. */
 	pvt->mouse_tracking_mode = MOUSE_TRACKING_NONE;
         pvt->mouse_pressed_buttons = 0;
+        pvt->mouse_handled_buttons = 0;
 	pvt->mouse_last_x = 0;
 	pvt->mouse_last_y = 0;
 	pvt->mouse_xterm_extension = FALSE;
