From c1e21719a2fff2fa9549f00053ac40173da54af9 Mon Sep 17 00:00:00 2001
From: Vaxry <vaxry@vaxry.net>
Date: Wed, 19 Jun 2024 18:36:40 +0200
Subject: [PATCH] core: avoid bumping hard rlimits, restore on fork

ref #6584
---
 src/Compositor.cpp              | 63 ++++++++++++++-------------------
 src/Compositor.hpp              |  8 +++--
 src/managers/KeybindManager.cpp |  1 +
 3 files changed, 34 insertions(+), 38 deletions(-)

diff --git a/src/Compositor.cpp b/src/Compositor.cpp
index 7777a55f..7a1d035d 100644
--- a/src/Compositor.cpp
+++ b/src/Compositor.cpp
@@ -71,44 +71,35 @@ void handleUserSignal(int sig) {
     }
 }
 
-static void bumpNofile() {
-    unsigned long limit = 1024;
-
-    try {
-        std::ifstream f("/proc/sys/fs/nr_open");
-        if (!f.good())
-            limit = 1073741816;
-        else {
-            std::string content((std::istreambuf_iterator<char>(f)), (std::istreambuf_iterator<char>()));
-            f.close();
-
-            limit = std::stoll(content);
-        }
-
-    } catch (...) { limit = 1073741816; }
-
-    struct rlimit rlimit_;
-    if (!getrlimit(RLIMIT_NOFILE, &rlimit_))
-        Debug::log(LOG, "Old rlimit: soft -> {}, hard -> {}", rlimit_.rlim_cur, rlimit_.rlim_max);
-
-    if (rlimit_.rlim_max <= 1024)
-        rlimit_.rlim_max = limit;
-
-    unsigned long oldHardLimit = rlimit_.rlim_max;
-
-    rlimit_.rlim_max = limit;
-
-    if (setrlimit(RLIMIT_NOFILE, &rlimit_) < 0) {
-        Debug::log(LOG, "Failed bumping NOFILE limits higher, retrying with previous hard.");
-        rlimit_.rlim_max = oldHardLimit;
-        rlimit_.rlim_cur = std::clamp((unsigned long)limit, 1UL, (unsigned long)rlimit_.rlim_max);
-
-        if (setrlimit(RLIMIT_NOFILE, &rlimit_) < 0)
-            Debug::log(LOG, "Failed bumping NOFILE limits higher for the second time.");
+void CCompositor::bumpNofile() {
+    if (!getrlimit(RLIMIT_NOFILE, &m_sOriginalNofile))
+        Debug::log(LOG, "Old rlimit: soft -> {}, hard -> {}", m_sOriginalNofile.rlim_cur, m_sOriginalNofile.rlim_max);
+    else {
+        Debug::log(ERR, "Failed to get NOFILE rlimits");
+        m_sOriginalNofile.rlim_max = 0;
+        return;
     }
 
-    if (!getrlimit(RLIMIT_NOFILE, &rlimit_))
-        Debug::log(LOG, "New rlimit: soft -> {}, hard -> {}", rlimit_.rlim_cur, rlimit_.rlim_max);
+    rlimit newLimit = m_sOriginalNofile;
+
+    newLimit.rlim_cur = newLimit.rlim_max;
+
+    if (setrlimit(RLIMIT_NOFILE, &newLimit) < 0) {
+        Debug::log(ERR, "Failed bumping NOFILE limits higher");
+        m_sOriginalNofile.rlim_max = 0;
+        return;
+    }
+
+    if (!getrlimit(RLIMIT_NOFILE, &newLimit))
+        Debug::log(LOG, "New rlimit: soft -> {}, hard -> {}", newLimit.rlim_cur, newLimit.rlim_max);
+}
+
+void CCompositor::restoreNofile() {
+    if (m_sOriginalNofile.rlim_max <= 0)
+        return;
+
+    if (setrlimit(RLIMIT_NOFILE, &m_sOriginalNofile) < 0)
+        Debug::log(ERR, "Failed restoring NOFILE limits");
 }
 
 CCompositor::CCompositor() {
diff --git a/src/Compositor.hpp b/src/Compositor.hpp
index 242c3b13..5a1d8a64 100644
--- a/src/Compositor.hpp
+++ b/src/Compositor.hpp
@@ -3,6 +3,7 @@
 #include <memory>
 #include <deque>
 #include <list>
+#include <sys/resource.h>
 
 #include "defines.hpp"
 #include "debug/Log.hpp"
@@ -81,6 +82,8 @@ class CCompositor {
     void                                      cleanup();
     void                                      createLockFile();
     void                                      removeLockFile();
+    void                                      bumpNofile();
+    void                                      restoreNofile();
 
     WP<CWLSurfaceResource>                    m_pLastFocus;
     PHLWINDOWREF                              m_pLastWindow;
@@ -190,8 +193,9 @@ class CCompositor {
     void             initManagers(eManagersInitStage stage);
     void             prepareFallbackOutput();
 
-    uint64_t         m_iHyprlandPID  = 0;
-    wl_event_source* m_critSigSource = nullptr;
+    uint64_t         m_iHyprlandPID    = 0;
+    wl_event_source* m_critSigSource   = nullptr;
+    rlimit           m_sOriginalNofile = {0};
 };
 
 inline std::unique_ptr<CCompositor> g_pCompositor;
diff --git a/src/managers/KeybindManager.cpp b/src/managers/KeybindManager.cpp
index 875ba239..80af767c 100644
--- a/src/managers/KeybindManager.cpp
+++ b/src/managers/KeybindManager.cpp
@@ -890,6 +890,7 @@ uint64_t CKeybindManager::spawnRaw(std::string args) {
     }
     if (child == 0) {
         // run in child
+        g_pCompositor->restoreNofile();
 
         sigset_t set;
         sigemptyset(&set);