From f84421fd9d096b2e513b48d6248240a3acba4391 Mon Sep 17 00:00:00 2001
From: Travis Biehn <tbiehn@gmail.com>
Date: Tue, 31 Dec 2024 19:44:05 -0500
Subject: [PATCH] Modified os/file.zig to record original file handle
 softlimits, added new global to track them, and fix up child posix processes
 in pre_exec handler specified in Exec.zig. Addresses #4215

---
 src/global.zig      |  4 +++-
 src/os/file.zig     | 12 ++++++++----
 src/termio/Exec.zig | 16 ++++++++++++++++
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/src/global.zig b/src/global.zig
index 7e43a9184..e5f9d8c65 100644
--- a/src/global.zig
+++ b/src/global.zig
@@ -27,6 +27,7 @@ pub const GlobalState = struct {
     alloc: std.mem.Allocator,
     action: ?cli.Action,
     logging: Logging,
+    posixfilehandles: u64,
 
     /// The app resources directory, equivalent to zig-out/share when we build
     /// from source. This is null if we can't detect it.
@@ -56,6 +57,7 @@ pub const GlobalState = struct {
             .alloc = undefined,
             .action = null,
             .logging = .{ .stderr = {} },
+            .posixfilehandles = 0,
             .resources_dir = null,
         };
         errdefer self.deinit();
@@ -124,7 +126,7 @@ pub const GlobalState = struct {
         std.log.info("libxev backend={}", .{xev.backend});
 
         // First things first, we fix our file descriptors
-        internal_os.fixMaxFiles();
+        self.posixfilehandles = internal_os.fixMaxFiles();
 
         // Initialize our crash reporting.
         crash.init(self.alloc) catch |err| {
diff --git a/src/os/file.zig b/src/os/file.zig
index e0ec2f52c..3544f531a 100644
--- a/src/os/file.zig
+++ b/src/os/file.zig
@@ -7,18 +7,21 @@ const log = std.log.scoped(.os);
 /// This maximizes the number of file descriptors we can have open. We
 /// need to do this because each window consumes at least a handful of fds.
 /// This is extracted from the Zig compiler source code.
-pub fn fixMaxFiles() void {
-    if (!@hasDecl(posix.system, "rlimit")) return;
+/// Return zero if there's an error, otherwise returns the old softlimit.
+pub fn fixMaxFiles() u64 {
+    if (!@hasDecl(posix.system, "rlimit")) return 0;
 
     var lim = posix.getrlimit(.NOFILE) catch {
         log.warn("failed to query file handle limit, may limit max windows", .{});
-        return; // Oh well; we tried.
+        return 0; // Oh well; we tried.
     };
 
+    const presoft = lim.cur;
+
     // If we're already at the max, we're done.
     if (lim.cur >= lim.max) {
         log.debug("file handle limit already maximized value={}", .{lim.cur});
-        return;
+        return presoft;
     }
 
     // Do a binary search for the limit.
@@ -41,6 +44,7 @@ pub fn fixMaxFiles() void {
     }
 
     log.debug("file handle limit raised value={}", .{lim.cur});
+    return presoft;
 }
 
 /// Return the recommended path for temporary files.
diff --git a/src/termio/Exec.zig b/src/termio/Exec.zig
index 423ebfa28..01c7310a4 100644
--- a/src/termio/Exec.zig
+++ b/src/termio/Exec.zig
@@ -8,6 +8,7 @@ const builtin = @import("builtin");
 const assert = std.debug.assert;
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
+const global_state = &@import("../global.zig").state;
 const posix = std.posix;
 const xev = @import("xev");
 const build_config = @import("../build_config.zig");
@@ -1160,6 +1161,21 @@ const Subprocess = struct {
                         "error initializing child: {}",
                         .{err},
                     );
+                    // if posixfilehandles > 0 restore old file handle limit to child process.
+                    if (global_state.posixfilehandles > 0) {
+                        // restore file handles here.
+                        // Restore SOFT limits.
+                        var lim = posix.getrlimit(.NOFILE) catch {
+                            log.err("failed to query file handle limit in child process", .{});
+                            return;
+                        };
+
+                        lim.cur = global_state.posixfilehandles;
+
+                        if (posix.setrlimit(.NOFILE, lim)) |_| {} else |_| {
+                            log.err("failed to restore file handle limits in child process.", .{});
+                        }
+                    }
                 }
             }).callback,
             .data = self,