renderer/metal: wip for loading custom shader pipelines

src/renderer/Metal.zig

@@ -257,8 +257,8 @@ pub fn init(alloc: Allocator, options: renderer.Options) !Metal {
     errdefer buf_instance.deinit();
 
     // Initialize our shaders
-    var shaders = try Shaders.init(device);
+    var shaders = try Shaders.init(alloc, device, &.{});
-    errdefer shaders.deinit();
+    errdefer shaders.deinit(alloc);
 
     // Font atlas textures
     const texture_greyscale = try initAtlasTexture(device, &options.font_group.atlas_greyscale);
@@ -328,7 +328,7 @@ pub fn deinit(self: *Metal) void {
     deinitMTLResource(self.texture_color);
     self.queue.msgSend(void, objc.sel("release"), .{});
 
-    self.shaders.deinit();
+    self.shaders.deinit(self.alloc);
 
     self.* = undefined;
 }

src/renderer/metal/shaders.zig

@@ -12,10 +12,31 @@ const log = std.log.scoped(.metal);
 /// This contains the state for the shaders used by the Metal renderer.
 pub const Shaders = struct {
     library: objc.Object,
+
+    /// The cell shader is the shader used to render the terminal cells.
+    /// It is a single shader that is used for both the background and
+    /// foreground.
     cell_pipeline: objc.Object,
+
+    /// The image shader is the shader used to render images for things
+    /// like the Kitty image protocol.
     image_pipeline: objc.Object,
 
-    pub fn init(device: objc.Object) !Shaders {
+    /// Custom shaders to run against the final drawable texture. This
+    /// can be used to apply a lot of effects. Each shader is run in sequence
+    /// against the output of the previous shader.
+    post_pipelines: []const objc.Object,
+
+    /// Initialize our shader set.
+    ///
+    /// "post_shaders" is an optional list of postprocess shaders to run
+    /// against the final drawable texture. This is an array of shader source
+    /// code, not file paths.
+    pub fn init(
+        alloc: Allocator,
+        device: objc.Object,
+        post_shaders: []const [:0]const u8,
+    ) !Shaders {
         const library = try initLibrary(device);
         errdefer library.msgSend(void, objc.sel("release"), .{});
 
@@ -25,17 +46,43 @@ pub const Shaders = struct {
         const image_pipeline = try initImagePipeline(device, library);
         errdefer image_pipeline.msgSend(void, objc.sel("release"), .{});
 
+        const post_pipelines: []const objc.Object = initPostPipelines(
+            alloc,
+            device,
+            post_shaders,
+        ) catch |err| err: {
+            // If an error happens while building postprocess shaders we
+            // want to just not use any postprocess shaders since we don't
+            // want to block Ghostty from working.
+            log.warn("error initializing postprocess shaders err={}", .{err});
+            break :err &.{};
+        };
+        errdefer if (post_pipelines.len > 0) {
+            for (post_pipelines) |pipeline| pipeline.msgSend(void, objc.sel("release"), .{});
+            alloc.free(post_pipelines);
+        };
+
         return .{
             .library = library,
             .cell_pipeline = cell_pipeline,
             .image_pipeline = image_pipeline,
+            .post_pipelines = post_pipelines,
         };
     }
 
-    pub fn deinit(self: *Shaders) void {
+    pub fn deinit(self: *Shaders, alloc: Allocator) void {
+        // Release our primary shaders
         self.cell_pipeline.msgSend(void, objc.sel("release"), .{});
         self.image_pipeline.msgSend(void, objc.sel("release"), .{});
         self.library.msgSend(void, objc.sel("release"), .{});
+
+        // Release our postprocess shaders
+        if (self.post_pipelines.len > 0) {
+            for (self.post_pipelines) |pipeline| {
+                pipeline.msgSend(void, objc.sel("release"), .{});
+            }
+            alloc.free(self.post_pipelines);
+        }
     }
 };
 
@@ -105,6 +152,68 @@ fn initLibrary(device: objc.Object) !objc.Object {
     return library;
 }
 
+/// Initialize our custom shader pipelines. The shaders argument is a
+/// set of shader source code, not file paths.
+fn initPostPipelines(
+    alloc: Allocator,
+    device: objc.Object,
+    shaders: []const [:0]const u8,
+) ![]const objc.Object {
+    // If we have no shaders, do nothing.
+    if (shaders.len == 0) return &.{};
+
+    // Keeps track of how many shaders we successfully wrote.
+    var i: usize = 0;
+
+    // Initialize our result set. If any error happens, we undo everything.
+    var pipelines = try alloc.alloc(objc.Object, shaders.len);
+    errdefer {
+        for (pipelines[0..i]) |pipeline| {
+            pipeline.msgSend(void, objc.sel("release"), .{});
+        }
+        alloc.free(pipelines);
+    }
+
+    // Build each shader. Note we don't use "0.." to build our index
+    // because we need to keep track of our length to clean up above.
+    for (shaders) |source| {
+        pipelines[i] = try initPostPipeline(device, source);
+        i += 1;
+    }
+
+    return pipelines;
+}
+
+/// Initialize a single custom shader pipeline from shader source.
+fn initPostPipeline(device: objc.Object, data: [:0]const u8) !objc.Object {
+    // Create our library which has the shader source
+    const library = library: {
+        const source = try macos.foundation.String.createWithBytes(
+            data,
+            .utf8,
+            false,
+        );
+        defer source.release();
+
+        var err: ?*anyopaque = null;
+        const library = device.msgSend(
+            objc.Object,
+            objc.sel("newLibraryWithSource:options:error:"),
+            .{ source, @as(?*anyopaque, null), &err },
+        );
+        try checkError(err);
+        errdefer library.msgSend(void, objc.sel("release"), .{});
+
+        break :library library;
+    };
+    // TODO: need to do this once we set the pipeline
+    //defer library.msgSend(void, objc.sel("release"), .{});
+
+    // TODO: need to implement the actual pipeline
+
+    return library;
+}
+
 /// Initialize the cell render pipeline for our shader library.
 fn initCellPipeline(device: objc.Object, library: objc.Object) !objc.Object {
     // Get our vertex and fragment functions