diff --git a/shared/cairo-util.h b/shared/cairo-util.h
index cce76710..7bcbc296 100644
--- a/shared/cairo-util.h
+++ b/shared/cairo-util.h
@@ -183,7 +183,13 @@ frame_pointer_motion(struct frame *frame, void *pointer, int x, int y);
 void
 frame_pointer_leave(struct frame *frame, void *pointer);
 
-/* May set:
+/* Call to indicate that a button has been pressed/released.  The return
+ * value for a button release will be the same as for the corresponding
+ * press.  This allows you to more easily track grabs.  If you want the
+ * actual location, simply keep the location from the last
+ * frame_pointer_motion call.
+ *
+ * May set:
  *	FRAME_STATUS_MINIMIZE
  *	FRAME_STATUS_MAXIMIZE
  *	FRAME_STATUS_CLOSE
diff --git a/shared/frame.c b/shared/frame.c
index 956e104e..a501649f 100644
--- a/shared/frame.c
+++ b/shared/frame.c
@@ -54,6 +54,13 @@ struct frame_button {
 	enum frame_status status_effect;
 };
 
+struct frame_pointer_button {
+	struct wl_list link;
+	uint32_t button;
+	enum theme_location press_location;
+	struct frame_button *frame_button;
+};
+
 struct frame_pointer {
 	struct wl_list link;
 	void *data;
@@ -61,7 +68,7 @@ struct frame_pointer {
 	int x, y;
 
 	struct frame_button *hover_button;
-	int active;
+	struct wl_list down_buttons;
 };
 
 struct frame_touch {
@@ -141,10 +148,6 @@ frame_button_leave(struct frame_button *button, struct frame_pointer *pointer)
 	button->hover_count--;
 	if (!button->hover_count)
 		button->frame->status |= FRAME_STATUS_REPAINT;
-
-	/* In this case, we won't get a release */
-	if (pointer->active)
-		button->press_count--;
 }
 
 static void
@@ -162,13 +165,23 @@ static void
 frame_button_release(struct frame_button *button)
 {
 	button->press_count--;
-	if (!button->press_count)
-		button->frame->status |= FRAME_STATUS_REPAINT;
+	if (button->press_count)
+		return;
+
+	button->frame->status |= FRAME_STATUS_REPAINT;
 
 	if (!(button->flags & FRAME_BUTTON_CLICK_DOWN))
 		button->frame->status |= button->status_effect;
 }
 
+static void
+frame_button_cancel(struct frame_button *button)
+{
+	button->press_count--;
+	if (!button->press_count)
+		button->frame->status |= FRAME_STATUS_REPAINT;
+}
+
 static void
 frame_button_repaint(struct frame_button *button, cairo_t *cr)
 {
@@ -225,6 +238,7 @@ frame_pointer_get(struct frame *frame, void *data)
 		return NULL;
 
 	pointer->data = data;
+	wl_list_init(&pointer->down_buttons);
 	wl_list_insert(&frame->pointers, &pointer->link);
 
 	return pointer;
@@ -616,8 +630,6 @@ frame_pointer_motion(struct frame *frame, void *data, int x, int y)
 	if (pointer->hover_button)
 		frame_button_leave(pointer->hover_button, pointer);
 
-	/* No drags */
-	pointer->active = 0;
 	pointer->hover_button = button;
 
 	if (pointer->hover_button)
@@ -626,24 +638,97 @@ frame_pointer_motion(struct frame *frame, void *data, int x, int y)
 	return location;
 }
 
+static void
+frame_pointer_button_destroy(struct frame_pointer_button *button)
+{
+	wl_list_remove(&button->link);
+	free(button);
+}
+
+static void
+frame_pointer_button_press(struct frame *frame, struct frame_pointer *pointer,
+			   struct frame_pointer_button *button)
+{
+	if (button->button == BTN_RIGHT) {
+		if (button->press_location == THEME_LOCATION_TITLEBAR)
+			frame->status |= FRAME_STATUS_MENU;
+
+		frame_pointer_button_destroy(button);
+
+	} else if (button->button == BTN_LEFT) {
+		if (pointer->hover_button) {
+			frame_button_press(pointer->hover_button);
+		} else {
+			switch (button->press_location) {
+			case THEME_LOCATION_TITLEBAR:
+				frame->status |= FRAME_STATUS_MOVE;
+
+				frame_pointer_button_destroy(button);
+				break;
+			case THEME_LOCATION_RESIZING_TOP:
+			case THEME_LOCATION_RESIZING_BOTTOM:
+			case THEME_LOCATION_RESIZING_LEFT:
+			case THEME_LOCATION_RESIZING_RIGHT:
+			case THEME_LOCATION_RESIZING_TOP_LEFT:
+			case THEME_LOCATION_RESIZING_TOP_RIGHT:
+			case THEME_LOCATION_RESIZING_BOTTOM_LEFT:
+			case THEME_LOCATION_RESIZING_BOTTOM_RIGHT:
+				frame->status |= FRAME_STATUS_RESIZE;
+
+				frame_pointer_button_destroy(button);
+				break;
+			default:
+				break;
+			}
+		}
+	}
+}
+
+static void
+frame_pointer_button_release(struct frame *frame, struct frame_pointer *pointer,
+			     struct frame_pointer_button *button)
+{
+	if (button->button == BTN_LEFT && button->frame_button) {
+		if (button->frame_button == pointer->hover_button)
+			frame_button_release(button->frame_button);
+		else
+			frame_button_cancel(button->frame_button);
+	}
+}
+
+static void
+frame_pointer_button_cancel(struct frame *frame, struct frame_pointer *pointer,
+			    struct frame_pointer_button *button)
+{
+	if (button->frame_button)
+		frame_button_cancel(button->frame_button);
+}
+
 void
 frame_pointer_leave(struct frame *frame, void *data)
 {
 	struct frame_pointer *pointer = frame_pointer_get(frame, data);
+	struct frame_pointer_button *button, *next;
 	if (!pointer)
 		return;
 
 	if (pointer->hover_button)
 		frame_button_leave(pointer->hover_button, pointer);
 
+	wl_list_for_each_safe(button, next, &pointer->down_buttons, link) {
+		frame_pointer_button_cancel(frame, pointer, button);
+		frame_pointer_button_destroy(button);
+	}
+
 	frame_pointer_destroy(pointer);
 }
 
 enum theme_location
 frame_pointer_button(struct frame *frame, void *data,
-		     uint32_t button, enum frame_button_state state)
+		     uint32_t btn, enum frame_button_state state)
 {
 	struct frame_pointer *pointer = frame_pointer_get(frame, data);
+	struct frame_pointer_button *button;
 	enum theme_location location;
 
 	location = theme_get_location(frame->theme, pointer->x, pointer->y,
@@ -654,40 +739,29 @@ frame_pointer_button(struct frame *frame, void *data,
 	if (!pointer)
 		return location;
 
-	if (button == BTN_RIGHT) {
-		if (state == FRAME_BUTTON_PRESSED &&
-		    location == THEME_LOCATION_TITLEBAR)
-			frame->status |= FRAME_STATUS_MENU;
-
-	} else if (button == BTN_LEFT && state == FRAME_BUTTON_PRESSED) {
-		if (pointer->hover_button) {
-			pointer->active = 1;
-			frame_button_press(pointer->hover_button);
+	if (state == FRAME_BUTTON_PRESSED) {
+		button = malloc(sizeof *button);
+		if (!button)
 			return location;
-		} else {
-			switch (location) {
-			case THEME_LOCATION_TITLEBAR:
-				frame->status |= FRAME_STATUS_MOVE;
-				break;
-			case THEME_LOCATION_RESIZING_TOP:
-			case THEME_LOCATION_RESIZING_BOTTOM:
-			case THEME_LOCATION_RESIZING_LEFT:
-			case THEME_LOCATION_RESIZING_RIGHT:
-			case THEME_LOCATION_RESIZING_TOP_LEFT:
-			case THEME_LOCATION_RESIZING_TOP_RIGHT:
-			case THEME_LOCATION_RESIZING_BOTTOM_LEFT:
-			case THEME_LOCATION_RESIZING_BOTTOM_RIGHT:
-				frame->status |= FRAME_STATUS_RESIZE;
-				break;
-			default:
+
+		button->button = btn;
+		button->press_location = location;
+		button->frame_button = pointer->hover_button;
+		wl_list_insert(&pointer->down_buttons, &button->link);
+
+		frame_pointer_button_press(frame, pointer, button);
+	} else if (state == FRAME_BUTTON_RELEASED) {
+		button = NULL;
+		wl_list_for_each(button, &pointer->down_buttons, link)
+			if (button->button == btn)
 				break;
-			}
-		}
-	} else if (button == BTN_LEFT && state == FRAME_BUTTON_RELEASED) {
-		if (pointer->hover_button && pointer->active)
-			frame_button_release(pointer->hover_button);
+		/* Make sure we didn't hit the end */
+		if (&button->link == &pointer->down_buttons)
+			return location;
 
-		pointer->active = 0;
+		location = button->press_location;
+		frame_pointer_button_release(frame, pointer, button);
+		frame_pointer_button_destroy(button);
 	}
 
 	return location;