diff --git a/backend/x11/backend.c b/backend/x11/backend.c
index f14bbbe6..845495e7 100644
--- a/backend/x11/backend.c
+++ b/backend/x11/backend.c
@@ -33,38 +33,6 @@ struct wlr_x11_output *get_x11_output_from_window_id(struct wlr_x11_backend *x11
 	return NULL;
 }
 
-void get_x11_output_layout_box(struct wlr_x11_backend *backend,
-		struct wlr_box *box) {
-	int min_x = INT_MAX, min_y = INT_MAX;
-	int max_x = INT_MIN, max_y = INT_MIN;
-
-	struct wlr_x11_output *output;
-	wl_list_for_each(output, &backend->outputs, link) {
-		struct wlr_output *wlr_output = &output->wlr_output;
-
-		int width, height;
-		wlr_output_effective_resolution(wlr_output, &width, &height);
-
-		if (wlr_output->lx < min_x) {
-			min_x = wlr_output->lx;
-		}
-		if (wlr_output->ly < min_y) {
-			min_y = wlr_output->ly;
-		}
-		if (wlr_output->lx + width > max_x) {
-			max_x = wlr_output->lx + width;
-		}
-		if (wlr_output->ly + height > max_y) {
-			max_y = wlr_output->ly + height;
-		}
-	}
-
-	box->x = min_x;
-	box->y = min_y;
-	box->width = max_x - min_x;
-	box->height = max_y - min_y;
-}
-
 static void handle_x11_event(struct wlr_x11_backend *x11,
 		xcb_generic_event_t *event) {
 	handle_x11_input_event(x11, event);
@@ -200,7 +168,6 @@ static bool backend_start(struct wlr_backend *backend) {
 #endif
 
 	wlr_signal_emit_safe(&x11->backend.events.new_input, &x11->keyboard_dev);
-	wlr_signal_emit_safe(&x11->backend.events.new_input, &x11->pointer_dev);
 
 	for (size_t i = 0; i < x11->requested_outputs; ++i) {
 		wlr_x11_output_create(&x11->backend);
@@ -221,11 +188,7 @@ static void backend_destroy(struct wlr_backend *backend) {
 		wlr_output_destroy(&output->wlr_output);
 	}
 
-	wlr_signal_emit_safe(&x11->pointer_dev.events.destroy, &x11->pointer_dev);
-	wlr_signal_emit_safe(&x11->keyboard_dev.events.destroy, &x11->keyboard_dev);
-
 	wlr_input_device_destroy(&x11->keyboard_dev);
-	wlr_input_device_destroy(&x11->pointer_dev);
 
 	wlr_signal_emit_safe(&backend->events.destroy, backend);
 
@@ -320,11 +283,6 @@ struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
 	wlr_keyboard_init(&x11->keyboard, &keyboard_impl);
 	x11->keyboard_dev.keyboard = &x11->keyboard;
 
-	wlr_input_device_init(&x11->pointer_dev, WLR_INPUT_DEVICE_POINTER,
-		&input_device_impl, "X11 pointer", 0, 0);
-	wlr_pointer_init(&x11->pointer, &pointer_impl);
-	x11->pointer_dev.pointer = &x11->pointer;
-
 	x11->display_destroy.notify = handle_display_destroy;
 	wl_display_add_destroy_listener(display, &x11->display_destroy);
 
diff --git a/backend/x11/input_device.c b/backend/x11/input_device.c
index 42b703cf..6e5ca592 100644
--- a/backend/x11/input_device.c
+++ b/backend/x11/input_device.c
@@ -33,25 +33,16 @@ static void x11_handle_pointer_position(struct wlr_x11_output *output,
 	struct wlr_x11_backend *x11 = output->x11;
 	struct wlr_output *wlr_output = &output->wlr_output;
 
-	struct wlr_box box = { .x = x, .y = y };
-	wlr_box_transform(&box, wlr_output->transform, wlr_output->width,
-		wlr_output->height, &box);
-	box.x /= wlr_output->scale;
-	box.y /= wlr_output->scale;
+	int output_width, output_height;
+	wlr_output_effective_resolution(wlr_output, &output_width, &output_height);
 
-	struct wlr_box layout_box;
-	get_x11_output_layout_box(x11, &layout_box);
-
-	double ox = wlr_output->lx / (double)layout_box.width;
-	double oy = wlr_output->ly / (double)layout_box.height;
-
-	struct wlr_event_pointer_motion_absolute wlr_event = {
-		.device = &x11->pointer_dev,
+	struct wlr_event_pointer_motion_absolute event = {
+		.device = &output->pointer_dev,
 		.time_msec = time,
-		.x = box.x / (double)layout_box.width + ox,
-		.y = box.y / (double)layout_box.height + oy,
+		.x = (double)x / output_width,
+		.y = (double)y / output_height,
 	};
-	wlr_signal_emit_safe(&x11->pointer.events.motion_absolute, &wlr_event);
+	wlr_signal_emit_safe(&output->pointer.events.motion_absolute, &event);
 
 	x11->time = time;
 }
@@ -78,17 +69,23 @@ void handle_x11_input_event(struct wlr_x11_backend *x11,
 	case XCB_BUTTON_PRESS: {
 		xcb_button_press_event_t *ev = (xcb_button_press_event_t *)event;
 
+		struct wlr_x11_output *output =
+			get_x11_output_from_window_id(x11, ev->event);
+		if (output == NULL) {
+			break;
+		}
+
 		if (ev->detail == XCB_BUTTON_INDEX_4 ||
 				ev->detail == XCB_BUTTON_INDEX_5) {
 			double delta = (ev->detail == XCB_BUTTON_INDEX_4 ? -15 : 15);
 			struct wlr_event_pointer_axis axis = {
-				.device = &x11->pointer_dev,
+				.device = &output->pointer_dev,
 				.time_msec = ev->time,
 				.source = WLR_AXIS_SOURCE_WHEEL,
 				.orientation = WLR_AXIS_ORIENTATION_VERTICAL,
 				.delta = delta,
 			};
-			wlr_signal_emit_safe(&x11->pointer.events.axis, &axis);
+			wlr_signal_emit_safe(&output->pointer.events.axis, &axis);
 			x11->time = ev->time;
 			break;
 		}
@@ -97,17 +94,23 @@ void handle_x11_input_event(struct wlr_x11_backend *x11,
 	case XCB_BUTTON_RELEASE: {
 		xcb_button_press_event_t *ev = (xcb_button_press_event_t *)event;
 
+		struct wlr_x11_output *output =
+			get_x11_output_from_window_id(x11, ev->event);
+		if (output == NULL) {
+			break;
+		}
+
 		if (ev->detail != XCB_BUTTON_INDEX_4 &&
 				ev->detail != XCB_BUTTON_INDEX_5) {
 			struct wlr_event_pointer_button button = {
-				.device = &x11->pointer_dev,
+				.device = &output->pointer_dev,
 				.time_msec = ev->time,
 				.button = xcb_button_to_wl(ev->detail),
 				.state = event->response_type == XCB_BUTTON_PRESS ?
 					WLR_BUTTON_PRESSED : WLR_BUTTON_RELEASED,
 			};
 
-			wlr_signal_emit_safe(&x11->pointer.events.button, &button);
+			wlr_signal_emit_safe(&output->pointer.events.button, &button);
 		}
 		x11->time = ev->time;
 		return;
diff --git a/backend/x11/output.c b/backend/x11/output.c
index 1f61c5e9..c619d226 100644
--- a/backend/x11/output.c
+++ b/backend/x11/output.c
@@ -1,6 +1,9 @@
+#define _POSIX_C_SOURCE 200809L
 #include <assert.h>
 #include <stdlib.h>
+#include <string.h>
 #include <wlr/interfaces/wlr_output.h>
+#include <wlr/interfaces/wlr_pointer.h>
 #include <wlr/util/log.h>
 #include "backend/x11.h"
 #include "util/signal.h"
@@ -59,6 +62,8 @@ static void output_destroy(struct wlr_output *wlr_output) {
 	struct wlr_x11_output *output = (struct wlr_x11_output *)wlr_output;
 	struct wlr_x11_backend *x11 = output->x11;
 
+	wlr_input_device_destroy(&output->pointer_dev);
+
 	wl_list_remove(&output->link);
 	wl_event_source_remove(output->frame_timer);
 	wlr_egl_destroy_surface(&x11->egl, output->surf);
@@ -155,11 +160,19 @@ struct wlr_output *wlr_x11_output_create(struct wlr_backend *backend) {
 	struct wl_event_loop *ev = wl_display_get_event_loop(x11->wl_display);
 	output->frame_timer = wl_event_loop_add_timer(ev, signal_frame, output);
 
+	wl_list_insert(&x11->outputs, &output->link);
+
 	wl_event_source_timer_update(output->frame_timer, output->frame_delay);
 	wlr_output_update_enabled(wlr_output, true);
 
-	wl_list_insert(&x11->outputs, &output->link);
+	wlr_input_device_init(&output->pointer_dev, WLR_INPUT_DEVICE_POINTER,
+		&input_device_impl, "X11 pointer", 0, 0);
+	wlr_pointer_init(&output->pointer, &pointer_impl);
+	output->pointer_dev.pointer = &output->pointer;
+	output->pointer_dev.output_name = strdup(wlr_output->name);
+
 	wlr_signal_emit_safe(&x11->backend.events.new_output, wlr_output);
+	wlr_signal_emit_safe(&x11->backend.events.new_input, &output->pointer_dev);
 
 	return wlr_output;
 }
diff --git a/include/backend/x11.h b/include/backend/x11.h
index 1d56fbe8..f5557343 100644
--- a/include/backend/x11.h
+++ b/include/backend/x11.h
@@ -24,6 +24,9 @@ struct wlr_x11_output {
 	xcb_window_t win;
 	EGLSurface surf;
 
+	struct wlr_pointer pointer;
+	struct wlr_input_device pointer_dev;
+
 	struct wl_event_source *frame_timer;
 	int frame_delay;
 };
@@ -43,9 +46,6 @@ struct wlr_x11_backend {
 	struct wlr_keyboard keyboard;
 	struct wlr_input_device keyboard_dev;
 
-	struct wlr_pointer pointer;
-	struct wlr_input_device pointer_dev;
-
 	struct wlr_egl egl;
 	struct wlr_renderer *renderer;
 	struct wl_event_source *event_source;
@@ -74,8 +74,6 @@ struct wlr_x11_backend {
 
 struct wlr_x11_output *get_x11_output_from_window_id(struct wlr_x11_backend *x11,
 	xcb_window_t window);
-void get_x11_output_layout_box(struct wlr_x11_backend *backend,
-	struct wlr_box *box);
 
 extern const struct wlr_keyboard_impl keyboard_impl;
 extern const struct wlr_pointer_impl pointer_impl;