diff --git a/include/wlr/xwayland.h b/include/wlr/xwayland.h
index 3507e547..7471bbc6 100644
--- a/include/wlr/xwayland.h
+++ b/include/wlr/xwayland.h
@@ -16,7 +16,6 @@ struct wlr_xwayland {
 	struct wl_client *client;
 	struct wl_event_source *sigusr1_source;
 	struct wl_listener client_destroy;
-	struct wl_listener display_destroy;
 	struct wlr_xwm *xwm;
 	struct wlr_xwayland_cursor *cursor;
 	int wm_fd[2], wl_fd[2];
@@ -28,6 +27,7 @@ struct wlr_xwayland {
 	int display;
 	int x_fd[2];
 	struct wl_event_source *x_fd_read_event[2];
+	struct wl_listener display_destroy;
 
 	bool lazy;
 
diff --git a/xwayland/xwayland.c b/xwayland/xwayland.c
index 22d5cdeb..51035f0e 100644
--- a/xwayland/xwayland.c
+++ b/xwayland/xwayland.c
@@ -133,6 +133,13 @@ static void xwayland_finish_server(struct wlr_xwayland *wlr_xwayland) {
 		return;
 	}
 
+	if (wlr_xwayland->x_fd_read_event[0]) {
+		wl_event_source_remove(wlr_xwayland->x_fd_read_event[0]);
+		wl_event_source_remove(wlr_xwayland->x_fd_read_event[1]);
+
+		wlr_xwayland->x_fd_read_event[0] = wlr_xwayland->x_fd_read_event[1] = NULL;
+	}
+
 	if (wlr_xwayland->cursor != NULL) {
 		free(wlr_xwayland->cursor);
 	}
@@ -162,6 +169,10 @@ static void xwayland_finish_server(struct wlr_xwayland *wlr_xwayland) {
 }
 
 static void xwayland_finish_display(struct wlr_xwayland *wlr_xwayland) {
+	if (!wlr_xwayland || wlr_xwayland->display == -1) {
+		return;
+	}
+
 	safe_close(wlr_xwayland->x_fd[0]);
 	safe_close(wlr_xwayland->x_fd[1]);
 	wlr_xwayland->x_fd[0] = wlr_xwayland->x_fd[1] = -1;