diff --git a/include/wlr/xwayland.h b/include/wlr/xwayland.h
index 8646e7ac..c4c5842c 100644
--- a/include/wlr/xwayland.h
+++ b/include/wlr/xwayland.h
@@ -18,14 +18,11 @@
 
 struct wlr_xwm;
 struct wlr_xwayland_cursor;
-struct wlr_gtk_primary_selection_device_manager;
 
-struct wlr_xwayland {
+struct wlr_xwayland_server {
 	pid_t pid;
 	struct wl_client *client;
 	struct wl_event_source *sigusr1_source;
-	struct wlr_xwm *xwm;
-	struct wlr_xwayland_cursor *cursor;
 	int wm_fd[2], wl_fd[2];
 
 	time_t server_start;
@@ -38,6 +35,31 @@ struct wlr_xwayland {
 	struct wl_event_source *x_fd_read_event[2];
 	bool lazy;
 
+	struct wl_display *wl_display;
+
+	struct {
+		struct wl_signal ready;
+		struct wl_signal destroy;
+	} events;
+
+	struct wl_listener client_destroy;
+	struct wl_listener display_destroy;
+
+	void *data;
+};
+
+struct wlr_xwayland_server_ready_event {
+	struct wlr_xwayland_server *server;
+	int wm_fd;
+};
+
+struct wlr_xwayland {
+	struct wlr_xwayland_server *server;
+	struct wlr_xwm *xwm;
+	struct wlr_xwayland_cursor *cursor;
+
+	const char *display_name;
+
 	struct wl_display *wl_display;
 	struct wlr_compositor *compositor;
 	struct wlr_seat *seat;
@@ -54,8 +76,9 @@ struct wlr_xwayland {
 	 */
 	int (*user_event_handler)(struct wlr_xwm *xwm, xcb_generic_event_t *event);
 
+	struct wl_listener server_ready;
+	struct wl_listener server_destroy;
 	struct wl_listener client_destroy;
-	struct wl_listener display_destroy;
 	struct wl_listener seat_destroy;
 
 	void *data;
@@ -192,7 +215,11 @@ struct wlr_xwayland_resize_event {
 	uint32_t edges;
 };
 
-/** Create an Xwayland server.
+struct wlr_xwayland_server *wlr_xwayland_server_create(
+	struct wl_display *display, bool lazy);
+void wlr_xwayland_server_destroy(struct wlr_xwayland_server *server);
+
+/** Create an Xwayland server and XWM.
  *
  * The server supports a lazy mode in which Xwayland is only started when a
  * client tries to connect.
diff --git a/include/xwayland/xwm.h b/include/xwayland/xwm.h
index 351fce44..2569092f 100644
--- a/include/xwayland/xwm.h
+++ b/include/xwayland/xwm.h
@@ -138,7 +138,7 @@ struct wlr_xwm {
 	struct wl_listener seat_drag_source_destroy;
 };
 
-struct wlr_xwm *xwm_create(struct wlr_xwayland *wlr_xwayland);
+struct wlr_xwm *xwm_create(struct wlr_xwayland *wlr_xwayland, int wm_fd);
 
 void xwm_destroy(struct wlr_xwm *xwm);
 
diff --git a/xwayland/meson.build b/xwayland/meson.build
index 12c69da4..6cd77189 100644
--- a/xwayland/meson.build
+++ b/xwayland/meson.build
@@ -54,6 +54,7 @@ wlr_files += files(
 	'selection/incoming.c',
 	'selection/outgoing.c',
 	'selection/selection.c',
+	'server.c',
 	'sockets.c',
 	'xwayland.c',
 	'xwm.c',
diff --git a/xwayland/server.c b/xwayland/server.c
new file mode 100644
index 00000000..0c76a389
--- /dev/null
+++ b/xwayland/server.c
@@ -0,0 +1,400 @@
+#define _POSIX_C_SOURCE 200809L
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdlib.h>
+#include <stdnoreturn.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <time.h>
+#include <unistd.h>
+#include <wayland-server-core.h>
+#include <wlr/util/log.h>
+#include <wlr/xwayland.h>
+#include "sockets.h"
+#include "util/signal.h"
+
+static void safe_close(int fd) {
+	if (fd >= 0) {
+		close(fd);
+	}
+}
+
+static int fill_arg(char ***argv, const char *fmt, ...) {
+	int len;
+	char **cur_arg = *argv;
+	va_list args;
+	va_start(args, fmt);
+	len = vsnprintf(NULL, 0, fmt, args) + 1;
+	va_end(args);
+	while (*cur_arg) {
+		cur_arg++;
+	}
+	*cur_arg = malloc(len);
+	if (!*cur_arg) {
+		return -1;
+	}
+	*argv = cur_arg;
+	va_start(args, fmt);
+	len = vsnprintf(*cur_arg, len, fmt, args);
+	va_end(args);
+	return len;
+}
+
+noreturn static void exec_xwayland(struct wlr_xwayland_server *server) {
+	if (!set_cloexec(server->x_fd[0], false) ||
+			!set_cloexec(server->x_fd[1], false) ||
+			!set_cloexec(server->wm_fd[1], false) ||
+			!set_cloexec(server->wl_fd[1], false)) {
+		_exit(EXIT_FAILURE);
+	}
+
+	/* Make Xwayland signal us when it's ready */
+	signal(SIGUSR1, SIG_IGN);
+
+	char *argv[] = {
+		"Xwayland", NULL /* display, e.g. :1 */,
+		"-rootless", "-terminate",
+		"-listen", NULL /* x_fd[0] */,
+		"-listen", NULL /* x_fd[1] */,
+		"-wm", NULL /* wm_fd[1] */,
+		NULL,
+	};
+	char **cur_arg = argv;
+
+	if (fill_arg(&cur_arg, ":%d", server->display) < 0 ||
+			fill_arg(&cur_arg, "%d", server->x_fd[0]) < 0 ||
+			fill_arg(&cur_arg, "%d", server->x_fd[1]) < 0 ||
+			fill_arg(&cur_arg, "%d", server->wm_fd[1]) < 0) {
+		wlr_log_errno(WLR_ERROR, "alloc/print failure");
+		_exit(EXIT_FAILURE);
+	}
+
+	char wayland_socket_str[16];
+	snprintf(wayland_socket_str, sizeof(wayland_socket_str), "%d", server->wl_fd[1]);
+	setenv("WAYLAND_SOCKET", wayland_socket_str, true);
+
+	wlr_log(WLR_INFO, "WAYLAND_SOCKET=%d Xwayland :%d -rootless -terminate -listen %d -listen %d -wm %d",
+		server->wl_fd[1], server->display, server->x_fd[0],
+		server->x_fd[1], server->wm_fd[1]);
+
+	// Closes stdout/stderr depending on log verbosity
+	enum wlr_log_importance verbosity = wlr_log_get_verbosity();
+	int devnull = open("/dev/null", O_WRONLY | O_CREAT | O_CLOEXEC, 0666);
+	if (devnull < 0) {
+		wlr_log_errno(WLR_ERROR, "XWayland: failed to open /dev/null");
+		_exit(EXIT_FAILURE);
+	}
+	if (verbosity < WLR_INFO) {
+		dup2(devnull, STDOUT_FILENO);
+	}
+	if (verbosity < WLR_ERROR) {
+		dup2(devnull, STDERR_FILENO);
+	}
+
+	// This returns if and only if the call fails
+	execvp("Xwayland", argv);
+
+	wlr_log_errno(WLR_ERROR, "failed to exec Xwayland");
+	close(devnull);
+	_exit(EXIT_FAILURE);
+}
+
+static void server_finish_process(struct wlr_xwayland_server *server) {
+	if (!server || server->display == -1) {
+		return;
+	}
+
+	if (server->x_fd_read_event[0]) {
+		wl_event_source_remove(server->x_fd_read_event[0]);
+		wl_event_source_remove(server->x_fd_read_event[1]);
+
+		server->x_fd_read_event[0] = server->x_fd_read_event[1] = NULL;
+	}
+
+	if (server->client) {
+		wl_list_remove(&server->client_destroy.link);
+		wl_client_destroy(server->client);
+	}
+	if (server->sigusr1_source) {
+		wl_event_source_remove(server->sigusr1_source);
+	}
+
+	safe_close(server->wl_fd[0]);
+	safe_close(server->wl_fd[1]);
+	safe_close(server->wm_fd[0]);
+	safe_close(server->wm_fd[1]);
+	memset(server, 0, offsetof(struct wlr_xwayland_server, display));
+	server->wl_fd[0] = server->wl_fd[1] = -1;
+	server->wm_fd[0] = server->wm_fd[1] = -1;
+
+	/* We do not kill the Xwayland process, it dies to broken pipe
+	 * after we close our side of the wm/wl fds. This is more reliable
+	 * than trying to kill something that might no longer be Xwayland.
+	 */
+}
+
+static void server_finish_display(struct wlr_xwayland_server *server) {
+	if (!server || server->display == -1) {
+		return;
+	}
+
+	safe_close(server->x_fd[0]);
+	safe_close(server->x_fd[1]);
+	server->x_fd[0] = server->x_fd[1] = -1;
+
+	wl_list_remove(&server->display_destroy.link);
+
+	unlink_display_sockets(server->display);
+	server->display = -1;
+	server->display_name[0] = '\0';
+}
+
+static bool server_start(struct wlr_xwayland_server *server);
+static bool server_start_lazy(struct wlr_xwayland_server *server);
+
+static void handle_client_destroy(struct wl_listener *listener, void *data) {
+	struct wlr_xwayland_server *server =
+		wl_container_of(listener, server, client_destroy);
+
+	if (server->sigusr1_source) {
+		// Xwayland failed to start, let the sigusr1 handler deal with it
+		return;
+	}
+
+	// Don't call client destroy: it's being destroyed already
+	server->client = NULL;
+	wl_list_remove(&server->client_destroy.link);
+
+	server_finish_process(server);
+
+	if (time(NULL) - server->server_start > 5) {
+		if (server->lazy) {
+			wlr_log(WLR_INFO, "Restarting Xwayland (lazy)");
+			server_start_lazy(server);
+		} else  {
+			wlr_log(WLR_INFO, "Restarting Xwayland");
+			server_start(server);
+		}
+	}
+}
+
+static void handle_display_destroy(struct wl_listener *listener, void *data) {
+	struct wlr_xwayland_server *server =
+		wl_container_of(listener, server, display_destroy);
+
+	// Don't call client destroy: the display is being destroyed, it's too late
+	if (server->client) {
+		server->client = NULL;
+		wl_list_remove(&server->client_destroy.link);
+	}
+
+	wlr_xwayland_server_destroy(server);
+}
+
+static int xserver_handle_ready(int signal_number, void *data) {
+	struct wlr_xwayland_server *server = data;
+
+	int stat_val = -1;
+	while (waitpid(server->pid, &stat_val, 0) < 0) {
+		if (errno == EINTR) {
+			continue;
+		}
+		wlr_log_errno(WLR_ERROR, "waitpid for Xwayland fork failed");
+		goto error;
+	}
+	if (stat_val) {
+		wlr_log(WLR_ERROR, "Xwayland startup failed, not setting up xwm");
+		goto error;
+	}
+	wlr_log(WLR_DEBUG, "Xserver is ready");
+
+	wl_event_source_remove(server->sigusr1_source);
+	server->sigusr1_source = NULL;
+
+	struct wlr_xwayland_server_ready_event event = {
+		.server = server,
+		.wm_fd = server->wm_fd[0],
+	};
+	wlr_signal_emit_safe(&server->events.ready, &event);
+	/* ready is a one-shot signal, fire and forget */
+	wl_signal_init(&server->events.ready);
+
+	return 1; /* wayland event loop dispatcher's count */
+
+error:
+	/* clean up */
+	server_finish_process(server);
+	server_finish_display(server);
+	return 1;
+}
+
+static bool server_start_display(struct wlr_xwayland_server *server,
+		struct wl_display *wl_display) {
+	server->display_destroy.notify = handle_display_destroy;
+	wl_display_add_destroy_listener(wl_display, &server->display_destroy);
+
+	server->display = open_display_sockets(server->x_fd);
+	if (server->display < 0) {
+		server_finish_display(server);
+		return false;
+	}
+
+	snprintf(server->display_name, sizeof(server->display_name),
+		":%d", server->display);
+	return true;
+}
+
+static bool server_start(struct wlr_xwayland_server *server) {
+	if (socketpair(AF_UNIX, SOCK_STREAM, 0, server->wl_fd) != 0 ||
+			socketpair(AF_UNIX, SOCK_STREAM, 0, server->wm_fd) != 0) {
+		wlr_log_errno(WLR_ERROR, "socketpair failed");
+		server_finish_process(server);
+		return false;
+	}
+	if (!set_cloexec(server->wl_fd[0], true) ||
+			!set_cloexec(server->wl_fd[1], true) ||
+			!set_cloexec(server->wm_fd[0], true) ||
+			!set_cloexec(server->wm_fd[1], true)) {
+		server_finish_process(server);
+		return false;
+	}
+
+	server->server_start = time(NULL);
+
+	server->client = wl_client_create(server->wl_display, server->wl_fd[0]);
+	if (!server->client) {
+		wlr_log_errno(WLR_ERROR, "wl_client_create failed");
+		server_finish_process(server);
+		return false;
+	}
+
+	server->wl_fd[0] = -1; /* not ours anymore */
+
+	server->client_destroy.notify = handle_client_destroy;
+	wl_client_add_destroy_listener(server->client, &server->client_destroy);
+
+	struct wl_event_loop *loop = wl_display_get_event_loop(server->wl_display);
+	server->sigusr1_source = wl_event_loop_add_signal(loop, SIGUSR1,
+		xserver_handle_ready, server);
+
+	server->pid = fork();
+	if (server->pid < 0) {
+		wlr_log_errno(WLR_ERROR, "fork failed");
+		server_finish_process(server);
+		return false;
+	} else if (server->pid == 0) {
+		/* Double-fork, but we need to forward SIGUSR1 once Xserver(1)
+		 * is ready, or error if there was one. */
+		pid_t ppid = getppid();
+		sigset_t sigset;
+		sigemptyset(&sigset);
+		sigaddset(&sigset, SIGUSR1);
+		sigaddset(&sigset, SIGCHLD);
+		sigprocmask(SIG_BLOCK, &sigset, NULL);
+
+		pid_t pid = fork();
+		if (pid < 0) {
+			wlr_log_errno(WLR_ERROR, "second fork failed");
+			_exit(EXIT_FAILURE);
+		} else if (pid == 0) {
+			exec_xwayland(server);
+		}
+
+		int sig;
+		sigwait(&sigset, &sig);
+		kill(ppid, SIGUSR1);
+		wlr_log(WLR_DEBUG, "sent SIGUSR1 to process %d", ppid);
+		if (sig == SIGCHLD) {
+			waitpid(pid, NULL, 0);
+			_exit(EXIT_FAILURE);
+		}
+
+		_exit(EXIT_SUCCESS);
+	}
+
+	/* close child fds */
+	/* remain managing x sockets for lazy start */
+	close(server->wl_fd[1]);
+	close(server->wm_fd[1]);
+	server->wl_fd[1] = server->wm_fd[1] = -1;
+
+	return true;
+}
+
+static int xwayland_socket_connected(int fd, uint32_t mask, void *data) {
+	struct wlr_xwayland_server *server = data;
+
+	wl_event_source_remove(server->x_fd_read_event[0]);
+	wl_event_source_remove(server->x_fd_read_event[1]);
+	server->x_fd_read_event[0] = server->x_fd_read_event[1] = NULL;
+
+	server_start(server);
+
+	return 0;
+}
+
+static bool server_start_lazy(struct wlr_xwayland_server *server) {
+	struct wl_event_loop *loop = wl_display_get_event_loop(server->wl_display);
+	server->x_fd_read_event[0] = wl_event_loop_add_fd(loop, server->x_fd[0],
+		WL_EVENT_READABLE, xwayland_socket_connected, server);
+	server->x_fd_read_event[1] = wl_event_loop_add_fd(loop, server->x_fd[1],
+		WL_EVENT_READABLE, xwayland_socket_connected, server);
+
+	return true;
+}
+
+void wlr_xwayland_server_destroy(struct wlr_xwayland_server *server) {
+	if (!server) {
+		return;
+	}
+
+	server_finish_process(server);
+	server_finish_display(server);
+	wlr_signal_emit_safe(&server->events.destroy, NULL);
+	free(server);
+}
+
+struct wlr_xwayland_server *wlr_xwayland_server_create(
+		struct wl_display *wl_display, bool lazy) {
+	struct wlr_xwayland_server *server =
+		calloc(1, sizeof(struct wlr_xwayland_server));
+	if (!server) {
+		return NULL;
+	}
+
+	server->wl_display = wl_display;
+	server->lazy = lazy;
+
+	server->x_fd[0] = server->x_fd[1] = -1;
+	server->wl_fd[0] = server->wl_fd[1] = -1;
+	server->wm_fd[0] = server->wm_fd[1] = -1;
+
+	wl_signal_init(&server->events.ready);
+	wl_signal_init(&server->events.destroy);
+
+	if (!server_start_display(server, wl_display)) {
+		goto error_alloc;
+	}
+
+	if (server->lazy) {
+		if (!server_start_lazy(server)) {
+			goto error_display;
+		}
+	} else {
+		if (!server_start(server)) {
+			goto error_display;
+		}
+	}
+
+	return server;
+
+error_display:
+	server_finish_display(server);
+error_alloc:
+	free(server);
+	return NULL;
+}
diff --git a/xwayland/xwayland.c b/xwayland/xwayland.c
index 77698f44..ae86d541 100644
--- a/xwayland/xwayland.c
+++ b/xwayland/xwayland.c
@@ -26,437 +26,103 @@ struct wlr_xwayland_cursor {
 	int32_t hotspot_y;
 };
 
-static void safe_close(int fd) {
-	if (fd >= 0) {
-		close(fd);
-	}
+static void handle_server_destroy(struct wl_listener *listener, void *data) {
+	struct wlr_xwayland *xwayland =
+		wl_container_of(listener, xwayland, server_destroy);
+	wlr_xwayland_destroy(xwayland);
 }
 
-static int fill_arg(char ***argv, const char *fmt, ...) {
-	int len;
-	char **cur_arg = *argv;
-	va_list args;
-	va_start(args, fmt);
-	len = vsnprintf(NULL, 0, fmt, args) + 1;
-	va_end(args);
-	while (*cur_arg) {
-		cur_arg++;
-	}
-	*cur_arg = malloc(len);
-	if (!*cur_arg) {
-		return -1;
-	}
-	*argv = cur_arg;
-	va_start(args, fmt);
-	len = vsnprintf(*cur_arg, len, fmt, args);
-	va_end(args);
-	return len;
-}
+static void handle_server_ready(struct wl_listener *listener, void *data) {
+	struct wlr_xwayland *xwayland =
+		wl_container_of(listener, xwayland, server_ready);
+	struct wlr_xwayland_server_ready_event *event = data;
 
-_Noreturn static void exec_xwayland(struct wlr_xwayland *wlr_xwayland) {
-	if (!set_cloexec(wlr_xwayland->x_fd[0], false) ||
-			!set_cloexec(wlr_xwayland->x_fd[1], false) ||
-			!set_cloexec(wlr_xwayland->wm_fd[1], false) ||
-			!set_cloexec(wlr_xwayland->wl_fd[1], false)) {
-		_exit(EXIT_FAILURE);
-	}
-
-	/* Make Xwayland signal us when it's ready */
-	signal(SIGUSR1, SIG_IGN);
-
-	char *argv[] = {
-		"Xwayland", NULL /* display, e.g. :1 */,
-		"-rootless", "-terminate",
-		"-listen", NULL /* x_fd[0] */,
-		"-listen", NULL /* x_fd[1] */,
-		"-wm", NULL /* wm_fd[1] */,
-		NULL,
-	};
-	char **cur_arg = argv;
-
-	if (fill_arg(&cur_arg, ":%d", wlr_xwayland->display) < 0 ||
-			fill_arg(&cur_arg, "%d", wlr_xwayland->x_fd[0]) < 0 ||
-			fill_arg(&cur_arg, "%d", wlr_xwayland->x_fd[1]) < 0 ||
-			fill_arg(&cur_arg, "%d", wlr_xwayland->wm_fd[1]) < 0) {
-		wlr_log_errno(WLR_ERROR, "alloc/print failure");
-		_exit(EXIT_FAILURE);
-	}
-
-	char wayland_socket_str[16];
-	snprintf(wayland_socket_str, sizeof(wayland_socket_str), "%d", wlr_xwayland->wl_fd[1]);
-	setenv("WAYLAND_SOCKET", wayland_socket_str, true);
-
-	wlr_log(WLR_INFO, "WAYLAND_SOCKET=%d Xwayland :%d -rootless -terminate -listen %d -listen %d -wm %d",
-		wlr_xwayland->wl_fd[1], wlr_xwayland->display, wlr_xwayland->x_fd[0],
-		wlr_xwayland->x_fd[1], wlr_xwayland->wm_fd[1]);
-
-	// Closes stdout/stderr depending on log verbosity
-	enum wlr_log_importance verbosity = wlr_log_get_verbosity();
-	int devnull = open("/dev/null", O_WRONLY | O_CREAT | O_CLOEXEC, 0666);
-	if (devnull < 0) {
-		wlr_log_errno(WLR_ERROR, "XWayland: failed to open /dev/null");
-		_exit(EXIT_FAILURE);
-	}
-	if (verbosity < WLR_INFO) {
-		dup2(devnull, STDOUT_FILENO);
-	}
-	if (verbosity < WLR_ERROR) {
-		dup2(devnull, STDERR_FILENO);
-	}
-
-	// This returns if and only if the call fails
-	execvp("Xwayland", argv);
-
-	wlr_log_errno(WLR_ERROR, "failed to exec Xwayland");
-	close(devnull);
-	_exit(EXIT_FAILURE);
-}
-
-static void xwayland_finish_server(struct wlr_xwayland *wlr_xwayland) {
-	if (!wlr_xwayland || wlr_xwayland->display == -1) {
+	xwayland->xwm = xwm_create(xwayland, event->wm_fd);
+	if (!xwayland->xwm) {
 		return;
 	}
 
-	if (wlr_xwayland->x_fd_read_event[0]) {
-		wl_event_source_remove(wlr_xwayland->x_fd_read_event[0]);
-		wl_event_source_remove(wlr_xwayland->x_fd_read_event[1]);
-
-		wlr_xwayland->x_fd_read_event[0] = wlr_xwayland->x_fd_read_event[1] = NULL;
+	if (xwayland->seat) {
+		xwm_set_seat(xwayland->xwm, xwayland->seat);
 	}
 
-	if (wlr_xwayland->cursor != NULL) {
-		free(wlr_xwayland->cursor);
-	}
-
-	xwm_destroy(wlr_xwayland->xwm);
-
-	if (wlr_xwayland->client) {
-		wl_list_remove(&wlr_xwayland->client_destroy.link);
-		wl_client_destroy(wlr_xwayland->client);
-	}
-	if (wlr_xwayland->sigusr1_source) {
-		wl_event_source_remove(wlr_xwayland->sigusr1_source);
-	}
-
-	safe_close(wlr_xwayland->wl_fd[0]);
-	safe_close(wlr_xwayland->wl_fd[1]);
-	safe_close(wlr_xwayland->wm_fd[0]);
-	safe_close(wlr_xwayland->wm_fd[1]);
-	memset(wlr_xwayland, 0, offsetof(struct wlr_xwayland, display));
-	wlr_xwayland->wl_fd[0] = wlr_xwayland->wl_fd[1] = -1;
-	wlr_xwayland->wm_fd[0] = wlr_xwayland->wm_fd[1] = -1;
-
-	/* We do not kill the Xwayland process, it dies to broken pipe
-	 * after we close our side of the wm/wl fds. This is more reliable
-	 * than trying to kill something that might no longer be Xwayland.
-	 */
-}
-
-static void xwayland_finish_display(struct wlr_xwayland *wlr_xwayland) {
-	if (!wlr_xwayland || wlr_xwayland->display == -1) {
-		return;
-	}
-
-	safe_close(wlr_xwayland->x_fd[0]);
-	safe_close(wlr_xwayland->x_fd[1]);
-	wlr_xwayland->x_fd[0] = wlr_xwayland->x_fd[1] = -1;
-
-	wl_list_remove(&wlr_xwayland->display_destroy.link);
-
-	unlink_display_sockets(wlr_xwayland->display);
-	wlr_xwayland->display = -1;
-	wlr_xwayland->display_name[0] = '\0';
-}
-
-static bool xwayland_start_server(struct wlr_xwayland *wlr_xwayland);
-static bool xwayland_start_server_lazy(struct wlr_xwayland *wlr_xwayland);
-
-static void handle_client_destroy(struct wl_listener *listener, void *data) {
-	struct wlr_xwayland *wlr_xwayland =
-		wl_container_of(listener, wlr_xwayland, client_destroy);
-
-	if (wlr_xwayland->sigusr1_source) {
-		// Xwayland failed to start, let the sigusr1 handler deal with it
-		return;
-	}
-
-	// Don't call client destroy: it's being destroyed already
-	wlr_xwayland->client = NULL;
-	wl_list_remove(&wlr_xwayland->client_destroy.link);
-
-	xwayland_finish_server(wlr_xwayland);
-
-	if (time(NULL) - wlr_xwayland->server_start > 5) {
-		if (wlr_xwayland->lazy) {
-			wlr_log(WLR_INFO, "Restarting Xwayland (lazy)");
-			xwayland_start_server_lazy(wlr_xwayland);
-		} else  {
-			wlr_log(WLR_INFO, "Restarting Xwayland");
-			xwayland_start_server(wlr_xwayland);
-		}
-	}
-}
-
-static void handle_display_destroy(struct wl_listener *listener, void *data) {
-	struct wlr_xwayland *wlr_xwayland =
-		wl_container_of(listener, wlr_xwayland, display_destroy);
-
-	// Don't call client destroy: the display is being destroyed, it's too late
-	if (wlr_xwayland->client) {
-		wlr_xwayland->client = NULL;
-		wl_list_remove(&wlr_xwayland->client_destroy.link);
-	}
-
-	wlr_xwayland_destroy(wlr_xwayland);
-}
-
-static int xserver_handle_ready(int signal_number, void *data) {
-	struct wlr_xwayland *wlr_xwayland = data;
-
-	int stat_val = -1;
-	while (waitpid(wlr_xwayland->pid, &stat_val, 0) < 0) {
-		if (errno == EINTR) {
-			continue;
-		}
-		wlr_log_errno(WLR_ERROR, "waitpid for Xwayland fork failed");
-		goto error;
-	}
-	if (stat_val) {
-		wlr_log(WLR_ERROR, "Xwayland startup failed, not setting up xwm");
-		goto error;
-	}
-	wlr_log(WLR_DEBUG, "Xserver is ready");
-
-	wlr_xwayland->xwm = xwm_create(wlr_xwayland);
-	if (!wlr_xwayland->xwm) {
-		goto error;
-	}
-
-	if (wlr_xwayland->seat) {
-		xwm_set_seat(wlr_xwayland->xwm, wlr_xwayland->seat);
-	}
-
-	wl_event_source_remove(wlr_xwayland->sigusr1_source);
-	wlr_xwayland->sigusr1_source = NULL;
-
-	if (wlr_xwayland->cursor != NULL) {
-		struct wlr_xwayland_cursor *cur = wlr_xwayland->cursor;
-		xwm_set_cursor(wlr_xwayland->xwm, cur->pixels, cur->stride, cur->width,
+	if (xwayland->cursor != NULL) {
+		struct wlr_xwayland_cursor *cur = xwayland->cursor;
+		xwm_set_cursor(xwayland->xwm, cur->pixels, cur->stride, cur->width,
 			cur->height, cur->hotspot_x, cur->hotspot_y);
 		free(cur);
-		wlr_xwayland->cursor = NULL;
+		xwayland->cursor = NULL;
 	}
 
-
-	wlr_signal_emit_safe(&wlr_xwayland->events.ready, wlr_xwayland);
+	wlr_signal_emit_safe(&xwayland->events.ready, NULL);
 	/* ready is a one-shot signal, fire and forget */
-	wl_signal_init(&wlr_xwayland->events.ready);
-
-	return 1; /* wayland event loop dispatcher's count */
-error:
-	/* clean up */
-	wlr_xwayland_set_seat(wlr_xwayland, NULL);
-	xwayland_finish_server(wlr_xwayland);
-	xwayland_finish_display(wlr_xwayland);
-
-	return 1;
+	wl_signal_init(&xwayland->events.ready);
 }
 
-static int xwayland_socket_connected(int fd, uint32_t mask, void* data){
-	struct wlr_xwayland *wlr_xwayland = data;
-
-	wl_event_source_remove(wlr_xwayland->x_fd_read_event[0]);
-	wl_event_source_remove(wlr_xwayland->x_fd_read_event[1]);
-
-	wlr_xwayland->x_fd_read_event[0] = wlr_xwayland->x_fd_read_event[1] = NULL;
-
-	xwayland_start_server(wlr_xwayland);
-
-	return 0;
-}
-
-static bool xwayland_start_display(struct wlr_xwayland *wlr_xwayland,
-		struct wl_display *wl_display) {
-
-	wlr_xwayland->display_destroy.notify = handle_display_destroy;
-	wl_display_add_destroy_listener(wl_display, &wlr_xwayland->display_destroy);
-
-	wlr_xwayland->display = open_display_sockets(wlr_xwayland->x_fd);
-	if (wlr_xwayland->display < 0) {
-		xwayland_finish_display(wlr_xwayland);
-		return false;
-	}
-
-	snprintf(wlr_xwayland->display_name, sizeof(wlr_xwayland->display_name),
-		":%d", wlr_xwayland->display);
-	return true;
-}
-
-static bool xwayland_start_server(struct wlr_xwayland *wlr_xwayland) {
-	if (socketpair(AF_UNIX, SOCK_STREAM, 0, wlr_xwayland->wl_fd) != 0 ||
-			socketpair(AF_UNIX, SOCK_STREAM, 0, wlr_xwayland->wm_fd) != 0) {
-		wlr_log_errno(WLR_ERROR, "socketpair failed");
-		xwayland_finish_server(wlr_xwayland);
-		return false;
-	}
-	if (!set_cloexec(wlr_xwayland->wl_fd[0], true) ||
-			!set_cloexec(wlr_xwayland->wl_fd[1], true) ||
-			!set_cloexec(wlr_xwayland->wm_fd[0], true) ||
-			!set_cloexec(wlr_xwayland->wm_fd[1], true)) {
-		xwayland_finish_server(wlr_xwayland);
-		return false;
-	}
-
-	wlr_xwayland->server_start = time(NULL);
-
-	wlr_xwayland->client =
-		wl_client_create(wlr_xwayland->wl_display, wlr_xwayland->wl_fd[0]);
-	if (!wlr_xwayland->client) {
-		wlr_log_errno(WLR_ERROR, "wl_client_create failed");
-		xwayland_finish_server(wlr_xwayland);
-		return false;
-	}
-
-	wlr_xwayland->wl_fd[0] = -1; /* not ours anymore */
-
-	wlr_xwayland->client_destroy.notify = handle_client_destroy;
-	wl_client_add_destroy_listener(wlr_xwayland->client,
-		&wlr_xwayland->client_destroy);
-
-	struct wl_event_loop *loop = wl_display_get_event_loop(wlr_xwayland->wl_display);
-	wlr_xwayland->sigusr1_source = wl_event_loop_add_signal(loop, SIGUSR1,
-		xserver_handle_ready, wlr_xwayland);
-
-	wlr_xwayland->pid = fork();
-	if (wlr_xwayland->pid < 0) {
-		wlr_log_errno(WLR_ERROR, "fork failed");
-		xwayland_finish_server(wlr_xwayland);
-		return false;
-	} else if (wlr_xwayland->pid == 0) {
-		/* Double-fork, but we need to forward SIGUSR1 once Xserver(1)
-		 * is ready, or error if there was one. */
-		pid_t ppid = getppid();
-		sigset_t sigset;
-		sigemptyset(&sigset);
-		sigaddset(&sigset, SIGUSR1);
-		sigaddset(&sigset, SIGCHLD);
-		sigprocmask(SIG_BLOCK, &sigset, NULL);
-
-		pid_t pid = fork();
-		if (pid < 0) {
-			wlr_log_errno(WLR_ERROR, "second fork failed");
-			_exit(EXIT_FAILURE);
-		} else if (pid == 0) {
-			exec_xwayland(wlr_xwayland);
-		}
-
-		int sig;
-		sigwait(&sigset, &sig);
-		kill(ppid, SIGUSR1);
-		wlr_log(WLR_DEBUG, "sent SIGUSR1 to process %d", ppid);
-		if (sig == SIGCHLD) {
-			waitpid(pid, NULL, 0);
-			_exit(EXIT_FAILURE);
-		}
-
-		_exit(EXIT_SUCCESS);
-	}
-
-	/* close child fds */
-	/* remain managing x sockets for lazy start */
-	close(wlr_xwayland->wl_fd[1]);
-	close(wlr_xwayland->wm_fd[1]);
-	wlr_xwayland->wl_fd[1] = wlr_xwayland->wm_fd[1] = -1;
-
-	return true;
-}
-
-static bool xwayland_start_server_lazy(struct wlr_xwayland *wlr_xwayland) {
-	struct wl_event_loop *loop = wl_display_get_event_loop(wlr_xwayland->wl_display);
-	wlr_xwayland->x_fd_read_event[0] =
-		wl_event_loop_add_fd(loop, wlr_xwayland->x_fd[0], WL_EVENT_READABLE,
-				xwayland_socket_connected, wlr_xwayland);
-	wlr_xwayland->x_fd_read_event[1] =
-		wl_event_loop_add_fd(loop, wlr_xwayland->x_fd[1], WL_EVENT_READABLE,
-				xwayland_socket_connected, wlr_xwayland);
-
-	return true;
-}
-
-void wlr_xwayland_destroy(struct wlr_xwayland *wlr_xwayland) {
-	if (!wlr_xwayland) {
+void wlr_xwayland_destroy(struct wlr_xwayland *xwayland) {
+	if (!xwayland) {
 		return;
 	}
 
-	wlr_xwayland_set_seat(wlr_xwayland, NULL);
-	xwayland_finish_server(wlr_xwayland);
-	xwayland_finish_display(wlr_xwayland);
-	free(wlr_xwayland);
+	wl_list_remove(&xwayland->server_destroy.link);
+	wl_list_remove(&xwayland->server_ready.link);
+
+	wlr_xwayland_set_seat(xwayland, NULL);
+	wlr_xwayland_server_destroy(xwayland->server);
+	free(xwayland);
 }
 
 struct wlr_xwayland *wlr_xwayland_create(struct wl_display *wl_display,
 		struct wlr_compositor *compositor, bool lazy) {
-	struct wlr_xwayland *wlr_xwayland = calloc(1, sizeof(struct wlr_xwayland));
-	if (!wlr_xwayland) {
+	struct wlr_xwayland *xwayland = calloc(1, sizeof(struct wlr_xwayland));
+	if (!xwayland) {
 		return NULL;
 	}
 
-	wlr_xwayland->wl_display = wl_display;
-	wlr_xwayland->compositor = compositor;
-	wlr_xwayland->lazy = lazy;
+	xwayland->wl_display = wl_display;
+	xwayland->compositor = compositor;
 
-	wlr_xwayland->x_fd[0] = wlr_xwayland->x_fd[1] = -1;
-	wlr_xwayland->wl_fd[0] = wlr_xwayland->wl_fd[1] = -1;
-	wlr_xwayland->wm_fd[0] = wlr_xwayland->wm_fd[1] = -1;
+	wl_signal_init(&xwayland->events.new_surface);
+	wl_signal_init(&xwayland->events.ready);
 
-	wl_signal_init(&wlr_xwayland->events.new_surface);
-	wl_signal_init(&wlr_xwayland->events.ready);
-
-	if (!xwayland_start_display(wlr_xwayland, wl_display)) {
-		goto error_alloc;
+	xwayland->server = wlr_xwayland_server_create(wl_display, lazy);
+	if (xwayland->server == NULL) {
+		free(xwayland->server);
+		return NULL;
 	}
 
-	if (wlr_xwayland->lazy) {
-		if (!xwayland_start_server_lazy(wlr_xwayland)) {
-			goto error_display;
-		}
-	} else {
-		if (!xwayland_start_server(wlr_xwayland)) {
-			goto error_display;
-		}
-	}
+	xwayland->display_name = xwayland->server->display_name;
 
-	return wlr_xwayland;
+	xwayland->server_destroy.notify = handle_server_destroy;
+	wl_signal_add(&xwayland->server->events.destroy, &xwayland->server_destroy);
 
-error_display:
-	xwayland_finish_display(wlr_xwayland);
+	xwayland->server_ready.notify = handle_server_ready;
+	wl_signal_add(&xwayland->server->events.ready, &xwayland->server_ready);
 
-error_alloc:
-	free(wlr_xwayland);
-	return NULL;
+	return xwayland;
 }
 
-void wlr_xwayland_set_cursor(struct wlr_xwayland *wlr_xwayland,
+void wlr_xwayland_set_cursor(struct wlr_xwayland *xwayland,
 		uint8_t *pixels, uint32_t stride, uint32_t width, uint32_t height,
 		int32_t hotspot_x, int32_t hotspot_y) {
-	if (wlr_xwayland->xwm != NULL) {
-		xwm_set_cursor(wlr_xwayland->xwm, pixels, stride, width, height,
+	if (xwayland->xwm != NULL) {
+		xwm_set_cursor(xwayland->xwm, pixels, stride, width, height,
 			hotspot_x, hotspot_y);
 		return;
 	}
 
-	free(wlr_xwayland->cursor);
+	free(xwayland->cursor);
 
-	wlr_xwayland->cursor = calloc(1, sizeof(struct wlr_xwayland_cursor));
-	if (wlr_xwayland->cursor == NULL) {
+	xwayland->cursor = calloc(1, sizeof(struct wlr_xwayland_cursor));
+	if (xwayland->cursor == NULL) {
 		return;
 	}
-	wlr_xwayland->cursor->pixels = pixels;
-	wlr_xwayland->cursor->stride = stride;
-	wlr_xwayland->cursor->width = width;
-	wlr_xwayland->cursor->height = height;
-	wlr_xwayland->cursor->hotspot_x = hotspot_x;
-	wlr_xwayland->cursor->hotspot_y = hotspot_y;
+	xwayland->cursor->pixels = pixels;
+	xwayland->cursor->stride = stride;
+	xwayland->cursor->width = width;
+	xwayland->cursor->height = height;
+	xwayland->cursor->hotspot_x = hotspot_x;
+	xwayland->cursor->hotspot_y = hotspot_y;
 }
 
 static void xwayland_handle_seat_destroy(struct wl_listener *listener,
diff --git a/xwayland/xwm.c b/xwayland/xwm.c
index b19efbcf..76175e5d 100644
--- a/xwayland/xwm.c
+++ b/xwayland/xwm.c
@@ -985,7 +985,7 @@ static void xwm_handle_surface_id_message(struct wlr_xwm *xwm,
 	/* Check if we got notified after wayland surface create event */
 	uint32_t id = ev->data.data32[0];
 	struct wl_resource *resource =
-		wl_client_get_object(xwm->xwayland->client, id);
+		wl_client_get_object(xwm->xwayland->server->client, id);
 	if (resource) {
 		struct wlr_surface *surface = wlr_surface_from_resource(resource);
 		xsurface->surface_id = 0;
@@ -1381,7 +1381,9 @@ static void handle_compositor_new_surface(struct wl_listener *listener,
 	struct wlr_xwm *xwm =
 		wl_container_of(listener, xwm, compositor_new_surface);
 	struct wlr_surface *surface = data;
-	if (wl_resource_get_client(surface->resource) != xwm->xwayland->client) {
+
+	struct wl_client *client = wl_resource_get_client(surface->resource);
+	if (client != xwm->xwayland->server->client) {
 		return;
 	}
 
@@ -1695,23 +1697,22 @@ void xwm_set_cursor(struct wlr_xwm *xwm, const uint8_t *pixels, uint32_t stride,
 	xcb_flush(xwm->xcb_conn);
 }
 
-struct wlr_xwm *xwm_create(struct wlr_xwayland *wlr_xwayland) {
+struct wlr_xwm *xwm_create(struct wlr_xwayland *xwayland, int wm_fd) {
 	struct wlr_xwm *xwm = calloc(1, sizeof(struct wlr_xwm));
 	if (xwm == NULL) {
 		return NULL;
 	}
 
-	xwm->xwayland = wlr_xwayland;
+	xwm->xwayland = xwayland;
 	wl_list_init(&xwm->surfaces);
 	wl_list_init(&xwm->unpaired_surfaces);
 	xwm->ping_timeout = 10000;
 
-	xwm->xcb_conn = xcb_connect_to_fd(wlr_xwayland->wm_fd[0], NULL);
+	xwm->xcb_conn = xcb_connect_to_fd(wm_fd, NULL);
 
 	int rc = xcb_connection_has_error(xwm->xcb_conn);
 	if (rc) {
 		wlr_log(WLR_ERROR, "xcb connect failed: %d", rc);
-		close(wlr_xwayland->wm_fd[0]);
 		free(xwm);
 		return NULL;
 	}
@@ -1723,18 +1724,15 @@ struct wlr_xwm *xwm_create(struct wlr_xwayland *wlr_xwayland) {
 		return NULL;
 	}
 #endif
+
 	xcb_screen_iterator_t screen_iterator =
 		xcb_setup_roots_iterator(xcb_get_setup(xwm->xcb_conn));
 	xwm->screen = screen_iterator.data;
 
-	struct wl_event_loop *event_loop = wl_display_get_event_loop(
-		wlr_xwayland->wl_display);
-	xwm->event_source =
-		wl_event_loop_add_fd(event_loop,
-			wlr_xwayland->wm_fd[0],
-			WL_EVENT_READABLE,
-			x11_event_handler,
-			xwm);
+	struct wl_event_loop *event_loop =
+		wl_display_get_event_loop(xwayland->wl_display);
+	xwm->event_source = wl_event_loop_add_fd(event_loop, wm_fd,
+		WL_EVENT_READABLE, x11_event_handler, xwm);
 	wl_event_source_check(xwm->event_source);
 
 	xwm_get_resources(xwm);
@@ -1781,10 +1779,10 @@ struct wlr_xwm *xwm_create(struct wlr_xwayland *wlr_xwayland) {
 	xwm_selection_init(xwm);
 
 	xwm->compositor_new_surface.notify = handle_compositor_new_surface;
-	wl_signal_add(&wlr_xwayland->compositor->events.new_surface,
+	wl_signal_add(&xwayland->compositor->events.new_surface,
 		&xwm->compositor_new_surface);
 	xwm->compositor_destroy.notify = handle_compositor_destroy;
-	wl_signal_add(&wlr_xwayland->compositor->events.destroy,
+	wl_signal_add(&xwayland->compositor->events.destroy,
 		&xwm->compositor_destroy);
 
 	xwm_create_wm_window(xwm);