diff --git a/types/data_device/wlr_drag.c b/types/data_device/wlr_drag.c
index 2b676fdc..558e9f22 100644
--- a/types/data_device/wlr_drag.c
+++ b/types/data_device/wlr_drag.c
@@ -487,12 +487,12 @@ bool seat_client_start_drag(struct wlr_seat_client *client,
 		drag_set_focus(drag, point->surface, point->sx, point->sy);
 	}
 
-	seat->drag = drag; // TODO: unset this thing somewhere
+	seat->drag = drag;
 	seat->drag_serial = serial;
 
-	if (seat->drag_source != NULL) {
-		wl_list_remove(&seat->drag_source_destroy.link);
-	}
+	// We need to destroy the previous source, because listeners only expect one
+	// active drag source at a time.
+	wlr_data_source_destroy(seat->drag_source);
 	seat->drag_source = source;
 	if (source != NULL) {
 		seat->drag_source_destroy.notify = seat_handle_drag_source_destroy;
diff --git a/xwayland/selection/dnd.c b/xwayland/selection/dnd.c
index fc090936..ec5f16c7 100644
--- a/xwayland/selection/dnd.c
+++ b/xwayland/selection/dnd.c
@@ -315,8 +315,6 @@ static void seat_handle_drag_source_destroy(struct wl_listener *listener,
 		wl_container_of(listener, xwm, seat_drag_source_destroy);
 
 	wl_list_remove(&xwm->seat_drag_source_destroy.link);
-	xwm->seat_drag_source_destroy.link.prev = NULL;
-	xwm->seat_drag_source_destroy.link.next = NULL;
 	xwm->drag_focus = NULL;
 }
 
@@ -334,9 +332,6 @@ void xwm_seat_handle_start_drag(struct wlr_xwm *xwm, struct wlr_drag *drag) {
 		wl_signal_add(&drag->events.destroy, &xwm->seat_drag_destroy);
 		xwm->seat_drag_destroy.notify = seat_handle_drag_destroy;
 
-		if (xwm->seat_drag_source_destroy.link.prev != NULL) {
-			wl_list_remove(&xwm->seat_drag_source_destroy.link);
-		}
 		wl_signal_add(&drag->source->events.destroy,
 			&xwm->seat_drag_source_destroy);
 		xwm->seat_drag_source_destroy.notify = seat_handle_drag_source_destroy;