xwayland: fix crash when trying to initialize without Xwayland installed (#9077)

src/Compositor.cpp

@@ -669,7 +669,7 @@ void CCompositor::initManagers(eManagersInitStage stage) {
             g_pDonationNagManager = std::make_unique<CDonationNagManager>();
 
             Debug::log(LOG, "Starting XWayland");
-            g_pXWayland = std::make_unique<CXWayland>(g_pCompositor->m_bEnableXwayland);
+            g_pXWayland = std::make_unique<CXWayland>(g_pCompositor->m_bWantsXwayland);
         } break;
         default: UNREACHABLE();
     }

src/Compositor.hpp

@@ -74,7 +74,7 @@ class CCompositor {
     bool                                       m_bIsShuttingDown = false;
     bool                                       m_bFinalRequests  = false;
     bool                                       m_bDesktopEnvSet  = false;
-    bool                                       m_bEnableXwayland = true;
+    bool                                       m_bWantsXwayland  = true;
 
     // ------------------------------------------------- //
 

src/config/ConfigManager.cpp

@@ -950,26 +950,16 @@ void CConfigManager::postConfigReload(const Hyprlang::CParseResult& result) {
     }
 
 #ifndef NO_XWAYLAND
-    const auto PENABLEXWAYLAND = std::any_cast<Hyprlang::INT>(m_pConfig->getConfigValue("xwayland:enabled"));
+    const auto PENABLEXWAYLAND      = std::any_cast<Hyprlang::INT>(m_pConfig->getConfigValue("xwayland:enabled"));
+    g_pCompositor->m_bWantsXwayland = PENABLEXWAYLAND;
     // enable/disable xwayland usage
     if (!isFirstLaunch) {
-        bool prevEnabledXwayland = g_pCompositor->m_bEnableXwayland;
+        bool prevEnabledXwayland = g_pXWayland->enabled();
-        if (PENABLEXWAYLAND != prevEnabledXwayland) {
+        if (g_pCompositor->m_bWantsXwayland != prevEnabledXwayland) {
-            g_pCompositor->m_bEnableXwayland = PENABLEXWAYLAND;
+            g_pXWayland = std::make_unique<CXWayland>(g_pCompositor->m_bWantsXwayland);
-            if (PENABLEXWAYLAND) {
-                Debug::log(LOG, "xwayland has been enabled");
-            } else {
-                Debug::log(LOG, "xwayland has been disabled, cleaning up...");
-                for (auto& w : g_pCompositor->m_vWindows) {
-                    if (w->m_pXDGSurface || !w->m_bIsX11)
-                        continue;
-                    g_pCompositor->closeWindow(w);
-                }
-            }
-            g_pXWayland = std::make_unique<CXWayland>(g_pCompositor->m_bEnableXwayland);
         }
     } else
-        g_pCompositor->m_bEnableXwayland = PENABLEXWAYLAND;
+        g_pCompositor->m_bWantsXwayland = PENABLEXWAYLAND;
 #endif
 
     if (!isFirstLaunch && !g_pCompositor->m_bUnsafeState)

src/xwayland/XWayland.cpp

@@ -1,21 +1,39 @@
 #include "XWayland.hpp"
+#include "../Compositor.hpp"
 #include "../debug/Log.hpp"
+#include "../helpers/fs/FsUtils.hpp"
 
-CXWayland::CXWayland(const bool enabled) {
+CXWayland::CXWayland(const bool wantsEnabled) {
 #ifndef NO_XWAYLAND
+    // Disable Xwayland and clean up if the user disabled it.
+    if (!wantsEnabled) {
+        Debug::log(LOG, "XWayland has been disabled, cleaning up...");
+        for (auto& w : g_pCompositor->m_vWindows) {
+            if (!w->m_bIsX11)
+                continue;
+            g_pCompositor->closeWindow(w);
+        }
+        unsetenv("DISPLAY");
+        m_enabled = false;
+        return;
+    }
+
+    if (!NFsUtils::executableExistsInPath("Xwayland")) {
+        // If Xwayland doesn't exist, don't try to start it.
+        Debug::log(LOG, "Unable to find XWayland; not starting it.");
+        return;
+    }
+
     Debug::log(LOG, "Starting up the XWayland server");
 
     pServer = std::make_unique<CXWaylandServer>();
 
-    if (!enabled) {
-        unsetenv("DISPLAY");
-        return;
-    }
-
     if (!pServer->create()) {
         Debug::log(ERR, "XWayland failed to start: it will not work.");
         return;
     }
+
+    m_enabled = true;
 #else
     Debug::log(LOG, "Not starting XWayland: disabled at compile time");
 #endif
@@ -31,3 +49,7 @@ void CXWayland::setCursor(unsigned char* pixData, uint32_t stride, const Vector2
     pWM->setCursor(pixData, stride, size, hotspot);
 #endif
 }
+
+bool CXWayland::enabled() {
+    return m_enabled;
+}

src/xwayland/XWayland.hpp

@@ -17,18 +17,22 @@ class CXWM;
 
 class CXWayland {
   public:
-    CXWayland(const bool enabled);
+    CXWayland(const bool wantsEnabled);
 
 #ifndef NO_XWAYLAND
     std::unique_ptr<CXWaylandServer> pServer;
     std::unique_ptr<CXWM>            pWM;
 #endif
+    bool enabled();
 
     void setCursor(unsigned char* pixData, uint32_t stride, const Vector2D& size, const Vector2D& hotspot);
 
     struct {
         CSignal newSurface;
     } events;
+
+  private:
+    bool m_enabled = false;
 };
 
 inline std::unique_ptr<CXWayland>                g_pXWayland;