core: if we change RLIMIT_NOFILE, reset it when executing commands (#4241)

Fixes #4232 .

src/Command.zig

@@ -18,6 +18,7 @@ const Command = @This();
 
 const std = @import("std");
 const builtin = @import("builtin");
+const global_state = &@import("global.zig").state;
 const internal_os = @import("os/main.zig");
 const windows = internal_os.windows;
 const TempDir = internal_os.TempDir;
@@ -175,6 +176,10 @@ fn startPosix(self: *Command, arena: Allocator) !void {
         // We don't log because that'll show up in the output.
     };
 
+    // Restore any rlimits that were set by Ghostty. This might fail but
+    // any failures are ignored (its best effort).
+    global_state.rlimits.restore();
+
     // If the user requested a pre exec callback, call it now.
     if (self.pre_exec) |f| f(self);
 

src/global.zig

@@ -27,6 +27,7 @@ pub const GlobalState = struct {
     alloc: std.mem.Allocator,
     action: ?cli.Action,
     logging: Logging,
+    rlimits: ResourceLimits = .{},
 
     /// The app resources directory, equivalent to zig-out/share when we build
     /// from source. This is null if we can't detect it.
@@ -56,6 +57,7 @@ pub const GlobalState = struct {
             .alloc = undefined,
             .action = null,
             .logging = .{ .stderr = {} },
+            .rlimits = .{},
             .resources_dir = null,
         };
         errdefer self.deinit();
@@ -123,8 +125,8 @@ pub const GlobalState = struct {
         std.log.info("renderer={}", .{renderer.Renderer});
         std.log.info("libxev backend={}", .{xev.backend});
 
-        // First things first, we fix our file descriptors
-        internal_os.fixMaxFiles();
+        // As early as possible, initialize our resource limits.
+        self.rlimits = ResourceLimits.init();
 
         // Initialize our crash reporting.
         crash.init(self.alloc) catch |err| {
@@ -174,3 +176,21 @@ pub const GlobalState = struct {
         }
     }
 };
+
+/// Maintains the Unix resource limits that we set for our process. This
+/// can be used to restore the limits to their original values.
+pub const ResourceLimits = struct {
+    nofile: ?internal_os.rlimit = null,
+
+    pub fn init() ResourceLimits {
+        return .{
+            // Maximize the number of file descriptors we can have open
+            // because we can consume a lot of them if we make many terminals.
+            .nofile = internal_os.fixMaxFiles(),
+        };
+    }
+
+    pub fn restore(self: *const ResourceLimits) void {
+        if (self.nofile) |lim| internal_os.restoreMaxFiles(lim);
+    }
+};

src/os/file.zig

@@ -4,24 +4,27 @@ const posix = std.posix;
 
 const log = std.log.scoped(.os);
 
+pub const rlimit = if (@hasDecl(posix.system, "rlimit")) posix.rlimit else struct {};
+
 /// This maximizes the number of file descriptors we can have open. We
 /// need to do this because each window consumes at least a handful of fds.
 /// This is extracted from the Zig compiler source code.
-pub fn fixMaxFiles() void {
-    if (!@hasDecl(posix.system, "rlimit")) return;
+pub fn fixMaxFiles() ?rlimit {
+    if (!@hasDecl(posix.system, "rlimit")) return null;
 
-    var lim = posix.getrlimit(.NOFILE) catch {
+    const old = posix.getrlimit(.NOFILE) catch {
         log.warn("failed to query file handle limit, may limit max windows", .{});
-        return; // Oh well; we tried.
+        return null; // Oh well; we tried.
     };
 
     // If we're already at the max, we're done.
-    if (lim.cur >= lim.max) {
-        log.debug("file handle limit already maximized value={}", .{lim.cur});
-        return;
+    if (old.cur >= old.max) {
+        log.debug("file handle limit already maximized value={}", .{old.cur});
+        return old;
     }
 
     // Do a binary search for the limit.
+    var lim = old;
     var min: posix.rlim_t = lim.cur;
     var max: posix.rlim_t = 1 << 20;
     // But if there's a defined upper bound, don't search, just set it.
@@ -41,6 +44,12 @@ pub fn fixMaxFiles() void {
     }
 
     log.debug("file handle limit raised value={}", .{lim.cur});
+    return old;
+}
+
+pub fn restoreMaxFiles(lim: rlimit) void {
+    if (!@hasDecl(posix.system, "rlimit")) return;
+    posix.setrlimit(.NOFILE, lim) catch {};
 }
 
 /// Return the recommended path for temporary files.

src/os/main.zig

@@ -32,7 +32,9 @@ pub const getenv = env.getenv;
 pub const setenv = env.setenv;
 pub const unsetenv = env.unsetenv;
 pub const launchedFromDesktop = desktop.launchedFromDesktop;
+pub const rlimit = file.rlimit;
 pub const fixMaxFiles = file.fixMaxFiles;
+pub const restoreMaxFiles = file.restoreMaxFiles;
 pub const allocTmpDir = file.allocTmpDir;
 pub const freeTmpDir = file.freeTmpDir;
 pub const isFlatpak = flatpak.isFlatpak;