diff --git a/src/vte-private.h b/src/vte-private.h
index 43d2266..e05216f 100644
--- a/src/vte-private.h
+++ b/src/vte-private.h
@@ -357,7 +357,7 @@ struct _VteTerminalPrivate {
 
 	int mouse_tracking_mode; /* this is of type MouseTrackingMode,
 				    but we need to guarantee its type. */
-	guint mouse_last_button;
+        guint mouse_pressed_buttons;      /* bits 0, 1, 2 resp. for buttons 1, 2, 3 */
 	long mouse_last_x, mouse_last_y;
 	gboolean mouse_autohide;
 	guint mouse_autoscroll_tag;
diff --git a/src/vte.c b/src/vte.c
index ee5288d..805d8d2 100644
--- a/src/vte.c
+++ b/src/vte.c
@@ -5508,6 +5508,9 @@ vte_terminal_feed_mouse_event(VteTerminal *terminal,
 
 	/* Encode the button information in cb. */
 	switch (button) {
+        case 0:                 /* No button, just dragging. */
+                cb = 3;
+                break;
 	case 1:			/* Left. */
 		cb = 0;
 		break;
@@ -5641,6 +5644,7 @@ vte_terminal_maybe_send_mouse_drag(VteTerminal *terminal, GdkEventMotion *event)
 	int height = terminal->pvt->char_height;
 	long col = ((long) event->x - terminal->pvt->padding.left) / width;
 	long row = ((long) event->y - terminal->pvt->padding.top) / height;
+        int button;
 
 	/* First determine if we even want to send notification. */
 	switch (event->type) {
@@ -5650,7 +5654,7 @@ vte_terminal_maybe_send_mouse_drag(VteTerminal *terminal, GdkEventMotion *event)
 
 		if (terminal->pvt->mouse_tracking_mode < MOUSE_TRACKING_ALL_MOTION_TRACKING) {
 
-			if (terminal->pvt->mouse_last_button == 0) {
+                        if (terminal->pvt->mouse_pressed_buttons == 0) {
 				return FALSE;
 			}
 			/* the xterm doc is not clear as to whether
@@ -5665,7 +5669,16 @@ vte_terminal_maybe_send_mouse_drag(VteTerminal *terminal, GdkEventMotion *event)
 		break;
 	}
 
-	vte_terminal_feed_mouse_event(terminal, terminal->pvt->mouse_last_button,
+        /* As per xterm, report the leftmost pressed button - if any. */
+        if (terminal->pvt->mouse_pressed_buttons & 1)
+                button = 1;
+        else if (terminal->pvt->mouse_pressed_buttons & 2)
+                button = 2;
+        else if (terminal->pvt->mouse_pressed_buttons & 4)
+                button = 3;
+        else
+                button = 0;
+        vte_terminal_feed_mouse_event(terminal, button,
 				      TRUE /* drag */, FALSE /* not release */,
 				      col, row);
 	return TRUE;
@@ -7024,7 +7037,7 @@ vte_terminal_motion_notify(GtkWidget *widget, GdkEventMotion *event)
 
 	vte_terminal_read_modifiers (terminal, (GdkEvent*) event);
 
-	if (terminal->pvt->mouse_last_button) {
+        if (terminal->pvt->mouse_pressed_buttons != 0) {
 		vte_terminal_match_hilite_hide (terminal);
 	} else {
 		/* Hilite any matches. */
@@ -7245,7 +7258,8 @@ vte_terminal_button_press(GtkWidget *widget, GdkEventButton *event)
 	}
 
 	/* Save the pointer state for later use. */
-	terminal->pvt->mouse_last_button = event->button;
+        if (event->button >= 1 && event->button <= 3)
+                terminal->pvt->mouse_pressed_buttons |= (1 << (event->button - 1));
 	terminal->pvt->mouse_last_x = x;
 	terminal->pvt->mouse_last_y = y;
 
@@ -7298,7 +7312,8 @@ vte_terminal_button_release(GtkWidget *widget, GdkEventButton *event)
 	}
 
 	/* Save the pointer state for later use. */
-	terminal->pvt->mouse_last_button = 0;
+        if (event->button >= 1 && event->button <= 3)
+                terminal->pvt->mouse_pressed_buttons &= ~(1 << (event->button - 1));
 	terminal->pvt->mouse_last_x = x;
 	terminal->pvt->mouse_last_y = y;
 	terminal->pvt->selecting_after_threshold = FALSE;
@@ -7358,7 +7373,7 @@ vte_terminal_focus_out(GtkWidget *widget, GdkEventFocus *event)
 		vte_terminal_match_hilite_hide (terminal);
 		/* Mark the cursor as invisible to disable hilite updating */
 		terminal->pvt->mouse_cursor_visible = FALSE;
-		terminal->pvt->mouse_last_button = 0;
+                terminal->pvt->mouse_pressed_buttons = 0;
 	}
 
 	terminal->pvt->has_focus = FALSE;
@@ -11952,7 +11967,7 @@ vte_terminal_reset(VteTerminal *terminal,
 	}
 	/* Reset mouse motion events. */
 	pvt->mouse_tracking_mode = MOUSE_TRACKING_NONE;
-	pvt->mouse_last_button = 0;
+        pvt->mouse_pressed_buttons = 0;
 	pvt->mouse_last_x = 0;
 	pvt->mouse_last_y = 0;
 	pvt->mouse_xterm_extension = FALSE;
