diff --git a/backend/wayland/output.c b/backend/wayland/output.c
index 18502ec9..87fd7bfd 100644
--- a/backend/wayland/output.c
+++ b/backend/wayland/output.c
@@ -36,6 +36,8 @@ static const uint32_t SUPPORTED_OUTPUT_STATE =
 
 static size_t last_output_num = 0;
 
+static const char *surface_tag = "wlr_wl_output";
+
 static struct wlr_wl_output *get_wl_output_from_output(
 		struct wlr_output *wlr_output) {
 	assert(wlr_output_is_wl(wlr_output));
@@ -43,6 +45,19 @@ static struct wlr_wl_output *get_wl_output_from_output(
 	return output;
 }
 
+struct wlr_wl_output *get_wl_output_from_surface(struct wlr_wl_backend *wl,
+		struct wl_surface *surface) {
+	if (wl_proxy_get_tag((struct wl_proxy *)surface) != &surface_tag) {
+		return NULL;
+	}
+	struct wlr_wl_output *output = wl_surface_get_user_data(surface);
+	assert(output != NULL);
+	if (output->backend != wl) {
+		return NULL;
+	}
+	return output;
+}
+
 static void surface_frame_callback(void *data, struct wl_callback *cb,
 		uint32_t time) {
 	struct wlr_wl_output *output = data;
@@ -774,6 +789,7 @@ struct wlr_output *wlr_wl_output_create(struct wlr_backend *wlr_backend) {
 		wlr_log_errno(WLR_ERROR, "Could not create output surface");
 		goto error;
 	}
+	wl_proxy_set_tag((struct wl_proxy *)output->surface, &surface_tag);
 	wl_surface_set_user_data(output->surface, output);
 	output->xdg_surface =
 		xdg_wm_base_get_xdg_surface(backend->xdg_wm_base, output->surface);
diff --git a/backend/wayland/pointer.c b/backend/wayland/pointer.c
index 4ce233e5..00f3254c 100644
--- a/backend/wayland/pointer.c
+++ b/backend/wayland/pointer.c
@@ -38,8 +38,10 @@ static void pointer_handle_enter(void *data, struct wl_pointer *wl_pointer,
 		return;
 	}
 
-	struct wlr_wl_output *output = wl_surface_get_user_data(surface);
-	assert(output);
+	struct wlr_wl_output *output = get_wl_output_from_surface(seat->backend, surface);
+	if (output == NULL) {
+		return;
+	}
 
 	struct wlr_wl_pointer *pointer = output_get_pointer(output, wl_pointer);
 	seat->active_pointer = pointer;
@@ -64,8 +66,10 @@ static void pointer_handle_leave(void *data, struct wl_pointer *wl_pointer,
 		return;
 	}
 
-	struct wlr_wl_output *output = wl_surface_get_user_data(surface);
-	assert(output);
+	struct wlr_wl_output *output = get_wl_output_from_surface(seat->backend, surface);
+	if (output == NULL) {
+		return;
+	}
 
 	if (seat->active_pointer != NULL &&
 			seat->active_pointer->output == output) {
diff --git a/backend/wayland/tablet_v2.c b/backend/wayland/tablet_v2.c
index d1aeecd2..b2b4a779 100644
--- a/backend/wayland/tablet_v2.c
+++ b/backend/wayland/tablet_v2.c
@@ -504,8 +504,13 @@ static void handle_tablet_tool_proximity_in(void *data,
 	struct tablet_tool *tool = data;
 	assert(tablet_id == tool->seat->zwp_tablet_v2);
 
+	struct wlr_wl_output *output = get_wl_output_from_surface(tool->seat->backend, surface);
+	if (output == NULL) {
+		return;
+	}
+
 	tool->is_in = true;
-	tool->output = wl_surface_get_user_data(surface);
+	tool->output = output;
 }
 
 static void handle_tablet_tool_proximity_out(void *data,
diff --git a/include/backend/wayland.h b/include/backend/wayland.h
index 1e731469..f28d1083 100644
--- a/include/backend/wayland.h
+++ b/include/backend/wayland.h
@@ -154,6 +154,8 @@ struct wlr_wl_seat {
 };
 
 struct wlr_wl_backend *get_wl_backend_from_backend(struct wlr_backend *backend);
+struct wlr_wl_output *get_wl_output_from_surface(struct wlr_wl_backend *wl,
+	struct wl_surface *surface);
 void update_wl_output_cursor(struct wlr_wl_output *output);
 
 void init_seat_keyboard(struct wlr_wl_seat *seat);