From 553d81afd16c12e687c1c84f9f1f7266c962d8a0 Mon Sep 17 00:00:00 2001
From: Mitchell Hashimoto <mitchell.hashimoto@gmail.com>
Date: Sun, 19 Nov 2023 21:39:30 -0800
Subject: [PATCH 1/7] terminal: enable kitty graphics commands on OpenGL

---
 src/terminal/kitty/graphics_exec.zig | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/terminal/kitty/graphics_exec.zig b/src/terminal/kitty/graphics_exec.zig
index 0415722b6..70c63db7e 100644
--- a/src/terminal/kitty/graphics_exec.zig
+++ b/src/terminal/kitty/graphics_exec.zig
@@ -35,12 +35,6 @@ pub fn execute(
         return null;
     }
 
-    // Only Metal supports rendering the images, right now.
-    if (comptime renderer.Renderer != renderer.Metal) {
-        log.warn("kitty graphics not supported on this renderer", .{});
-        return null;
-    }
-
     log.debug("executing kitty graphics command: quiet={} control={}", .{
         cmd.quiet,
         cmd.control,

From 76c76ce85ef5e675bfd598c153504fc7c7e6746d Mon Sep 17 00:00:00 2001
From: Mitchell Hashimoto <mitchell.hashimoto@gmail.com>
Date: Sun, 19 Nov 2023 22:08:07 -0800
Subject: [PATCH 2/7] renderer/opengl: upload kitty image textures

---
 pkg/opengl/Texture.zig        |   1 +
 src/renderer/OpenGL.zig       | 197 ++++++++++++++++++++++++++++++
 src/renderer/opengl/image.zig | 220 ++++++++++++++++++++++++++++++++++
 3 files changed, 418 insertions(+)
 create mode 100644 src/renderer/opengl/image.zig

diff --git a/pkg/opengl/Texture.zig b/pkg/opengl/Texture.zig
index afa22e926..87f3ca102 100644
--- a/pkg/opengl/Texture.zig
+++ b/pkg/opengl/Texture.zig
@@ -78,6 +78,7 @@ pub const InternalFormat = enum(c_int) {
 pub const Format = enum(c_uint) {
     red = c.GL_RED,
     rgb = c.GL_RGB,
+    rgba = c.GL_RGBA,
     bgra = c.GL_BGRA,
 
     // There are so many more that I haven't filled in.
diff --git a/src/renderer/OpenGL.zig b/src/renderer/OpenGL.zig
index 3d7031c8d..ab1ee27d0 100644
--- a/src/renderer/OpenGL.zig
+++ b/src/renderer/OpenGL.zig
@@ -22,7 +22,11 @@ const math = @import("../math.zig");
 const Surface = @import("../Surface.zig");
 
 const CellProgram = @import("opengl/CellProgram.zig");
+const gl_image = @import("opengl/image.zig");
 const custom = @import("opengl/custom.zig");
+const Image = gl_image.Image;
+const ImageMap = gl_image.ImageMap;
+const ImagePlacementList = std.ArrayListUnmanaged(gl_image.Placement);
 
 const log = std.log.scoped(.grid);
 
@@ -103,6 +107,12 @@ draw_mutex: DrawMutex = drawMutexZero,
 /// terminal is in reversed mode.
 draw_background: terminal.color.RGB,
 
+/// The images that we may render.
+images: ImageMap = .{},
+image_placements: ImagePlacementList = .{},
+image_bg_end: u32 = 0,
+image_text_end: u32 = 0,
+
 /// Defererred OpenGL operation to update the screen size.
 const SetScreenSize = struct {
     size: renderer.ScreenSize,
@@ -307,6 +317,13 @@ pub fn init(alloc: Allocator, options: renderer.Options) !OpenGL {
 pub fn deinit(self: *OpenGL) void {
     self.font_shaper.deinit();
 
+    {
+        var it = self.images.iterator();
+        while (it.next()) |kv| kv.value_ptr.deinit(self.alloc);
+        self.images.deinit(self.alloc);
+    }
+    self.image_placements.deinit(self.alloc);
+
     if (self.gl_state) |*v| v.deinit(self.alloc);
 
     self.cells.deinit(self.alloc);
@@ -623,6 +640,13 @@ pub fn updateFrame(
             cursor_blink_visible,
         );
 
+        // If we have Kitty graphics data, we enter a SLOW SLOW SLOW path.
+        // We only do this if the Kitty image state is dirty meaning only if
+        // it changes.
+        if (state.terminal.screen.kitty_images.dirty) {
+            try self.prepKittyGraphics(state.terminal);
+        }
+
         break :critical .{
             .gl_bg = self.background_color,
             .selection = selection,
@@ -651,6 +675,158 @@ pub fn updateFrame(
     }
 }
 
+/// This goes through the Kitty graphic placements and accumulates the
+/// placements we need to render on our viewport. It also ensures that
+/// the visible images are loaded on the GPU.
+fn prepKittyGraphics(
+    self: *OpenGL,
+    t: *terminal.Terminal,
+) !void {
+    const storage = &t.screen.kitty_images;
+    defer storage.dirty = false;
+
+    // We always clear our previous placements no matter what because
+    // we rebuild them from scratch.
+    self.image_placements.clearRetainingCapacity();
+
+    // Go through our known images and if there are any that are no longer
+    // in use then mark them to be freed.
+    //
+    // This never conflicts with the below because a placement can't
+    // reference an image that doesn't exist.
+    {
+        var it = self.images.iterator();
+        while (it.next()) |kv| {
+            if (storage.imageById(kv.key_ptr.*) == null) {
+                kv.value_ptr.markForUnload();
+            }
+        }
+    }
+
+    // The top-left and bottom-right corners of our viewport in screen
+    // points. This lets us determine offsets and containment of placements.
+    const top = (terminal.point.Viewport{}).toScreen(&t.screen);
+    const bot = (terminal.point.Viewport{
+        .x = t.screen.cols - 1,
+        .y = t.screen.rows - 1,
+    }).toScreen(&t.screen);
+
+    // Go through the placements and ensure the image is loaded on the GPU.
+    var it = storage.placements.iterator();
+    while (it.next()) |kv| {
+        // Find the image in storage
+        const p = kv.value_ptr;
+        const image = storage.imageById(kv.key_ptr.image_id) orelse {
+            log.warn(
+                "missing image for placement, ignoring image_id={}",
+                .{kv.key_ptr.image_id},
+            );
+            continue;
+        };
+
+        // If the selection isn't within our viewport then skip it.
+        const rect = p.rect(image, t);
+        if (rect.top_left.y > bot.y) continue;
+        if (rect.bottom_right.y < top.y) continue;
+
+        // If the top left is outside the viewport we need to calc an offset
+        // so that we render (0, 0) with some offset for the texture.
+        const offset_y: u32 = if (rect.top_left.y < t.screen.viewport) offset_y: {
+            const offset_cells = t.screen.viewport - rect.top_left.y;
+            const offset_pixels = offset_cells * self.cell_size.height;
+            break :offset_y @intCast(offset_pixels);
+        } else 0;
+
+        // If we already know about this image then do nothing
+        const gop = try self.images.getOrPut(self.alloc, kv.key_ptr.image_id);
+        if (!gop.found_existing) {
+            // Copy the data into the pending state.
+            const data = try self.alloc.dupe(u8, image.data);
+            errdefer self.alloc.free(data);
+
+            // Store it in the map
+            const pending: Image.Pending = .{
+                .width = image.width,
+                .height = image.height,
+                .data = data.ptr,
+            };
+
+            gop.value_ptr.* = switch (image.format) {
+                .rgb => .{ .pending_rgb = pending },
+                .rgba => .{ .pending_rgba = pending },
+                .png => unreachable, // should be decoded by now
+            };
+        }
+
+        // Convert our screen point to a viewport point
+        const viewport = p.point.toViewport(&t.screen);
+
+        // Calculate the source rectangle
+        const source_x = @min(image.width, p.source_x);
+        const source_y = @min(image.height, p.source_y + offset_y);
+        const source_width = if (p.source_width > 0)
+            @min(image.width - source_x, p.source_width)
+        else
+            image.width;
+        const source_height = if (p.source_height > 0)
+            @min(image.height, p.source_height)
+        else
+            image.height -| offset_y;
+
+        // Calculate the width/height of our image.
+        const dest_width = if (p.columns > 0) p.columns * self.cell_size.width else source_width;
+        const dest_height = if (p.rows > 0) p.rows * self.cell_size.height else source_height;
+
+        // Accumulate the placement
+        if (image.width > 0 and image.height > 0) {
+            try self.image_placements.append(self.alloc, .{
+                .image_id = kv.key_ptr.image_id,
+                .x = @intCast(p.point.x),
+                .y = @intCast(viewport.y),
+                .z = p.z,
+                .width = dest_width,
+                .height = dest_height,
+                .cell_offset_x = p.x_offset,
+                .cell_offset_y = p.y_offset,
+                .source_x = source_x,
+                .source_y = source_y,
+                .source_width = source_width,
+                .source_height = source_height,
+            });
+        }
+    }
+
+    // Sort the placements by their Z value.
+    std.mem.sortUnstable(
+        gl_image.Placement,
+        self.image_placements.items,
+        {},
+        struct {
+            fn lessThan(
+                ctx: void,
+                lhs: gl_image.Placement,
+                rhs: gl_image.Placement,
+            ) bool {
+                _ = ctx;
+                return lhs.z < rhs.z or (lhs.z == rhs.z and lhs.image_id < rhs.image_id);
+            }
+        }.lessThan,
+    );
+
+    // Find our indices
+    self.image_bg_end = 0;
+    self.image_text_end = 0;
+    const bg_limit = std.math.minInt(i32) / 2;
+    for (self.image_placements.items, 0..) |p, i| {
+        if (self.image_bg_end == 0 and p.z >= bg_limit) {
+            self.image_bg_end = @intCast(i);
+        }
+        if (self.image_text_end == 0 and p.z >= 0) {
+            self.image_text_end = @intCast(i);
+        }
+    }
+}
+
 /// rebuildCells rebuilds all the GPU cells from our CPU state. This is a
 /// slow operation but ensures that the GPU state exactly matches the CPU state.
 /// In steady-state operation, we use some GPU tricks to send down stale data
@@ -1396,6 +1572,27 @@ pub fn drawFrame(self: *OpenGL, surface: *apprt.Surface) !void {
     defer if (single_threaded_draw) self.draw_mutex.unlock();
     const gl_state: *GLState = if (self.gl_state) |*v| v else return;
 
+    // Go through our images and see if we need to setup any textures.
+    {
+        var image_it = self.images.iterator();
+        while (image_it.next()) |kv| {
+            switch (kv.value_ptr.*) {
+                .ready => {},
+
+                .pending_rgb,
+                .pending_rgba,
+                => try kv.value_ptr.upload(self.alloc),
+
+                .unload_pending,
+                .unload_ready,
+                => {
+                    kv.value_ptr.deinit(self.alloc);
+                    self.images.removeByPtr(kv.key_ptr);
+                },
+            }
+        }
+    }
+
     // Draw our terminal cells
     try self.drawCellProgram(gl_state);
 
diff --git a/src/renderer/opengl/image.zig b/src/renderer/opengl/image.zig
new file mode 100644
index 000000000..918149e93
--- /dev/null
+++ b/src/renderer/opengl/image.zig
@@ -0,0 +1,220 @@
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const gl = @import("opengl");
+
+/// Represents a single image placement on the grid. A placement is a
+/// request to render an instance of an image.
+pub const Placement = struct {
+    /// The image being rendered. This MUST be in the image map.
+    image_id: u32,
+
+    /// The grid x/y where this placement is located.
+    x: u32,
+    y: u32,
+    z: i32,
+
+    /// The width/height of the placed image.
+    width: u32,
+    height: u32,
+
+    /// The offset in pixels from the top left of the cell. This is
+    /// clamped to the size of a cell.
+    cell_offset_x: u32,
+    cell_offset_y: u32,
+
+    /// The source rectangle of the placement.
+    source_x: u32,
+    source_y: u32,
+    source_width: u32,
+    source_height: u32,
+};
+
+/// The map used for storing images.
+pub const ImageMap = std.AutoHashMapUnmanaged(u32, Image);
+
+/// The state for a single image that is to be rendered. The image can be
+/// pending upload or ready to use with a texture.
+pub const Image = union(enum) {
+    /// The image is pending upload to the GPU. The different keys are
+    /// different formats since some formats aren't accepted by the GPU
+    /// and require conversion.
+    ///
+    /// This data is owned by this union so it must be freed once the
+    /// image is uploaded.
+    pending_rgb: Pending,
+    pending_rgba: Pending,
+
+    /// The image is uploaded and ready to be used.
+    ready: gl.Texture,
+
+    /// The image is uploaded but is scheduled to be unloaded.
+    unload_pending: []u8,
+    unload_ready: gl.Texture,
+
+    /// Pending image data that needs to be uploaded to the GPU.
+    pub const Pending = struct {
+        height: u32,
+        width: u32,
+
+        /// Data is always expected to be (width * height * depth). Depth
+        /// is based on the union key.
+        data: [*]u8,
+
+        pub fn dataSlice(self: Pending, d: u32) []u8 {
+            return self.data[0..self.len(d)];
+        }
+
+        pub fn len(self: Pending, d: u32) u32 {
+            return self.width * self.height * d;
+        }
+    };
+
+    pub fn deinit(self: Image, alloc: Allocator) void {
+        switch (self) {
+            .pending_rgb => |p| alloc.free(p.dataSlice(3)),
+            .pending_rgba => |p| alloc.free(p.dataSlice(4)),
+            .unload_pending => |data| alloc.free(data),
+
+            .ready,
+            .unload_ready,
+            => |tex| tex.destroy(),
+        }
+    }
+
+    /// Mark this image for unload whatever state it is in.
+    pub fn markForUnload(self: *Image) void {
+        self.* = switch (self.*) {
+            .unload_pending,
+            .unload_ready,
+            => return,
+
+            .ready => |obj| .{ .unload_ready = obj },
+            .pending_rgb => |p| .{ .unload_pending = p.dataSlice(3) },
+            .pending_rgba => |p| .{ .unload_pending = p.dataSlice(4) },
+        };
+    }
+
+    /// Returns true if this image is pending upload.
+    pub fn isPending(self: Image) bool {
+        return self.pending() != null;
+    }
+
+    /// Returns true if this image is pending an unload.
+    pub fn isUnloading(self: Image) bool {
+        return switch (self) {
+            .unload_pending,
+            .unload_ready,
+            => true,
+
+            .ready,
+            .pending_rgb,
+            .pending_rgba,
+            => false,
+        };
+    }
+
+    /// Converts the image data to a format that can be uploaded to the GPU.
+    /// If the data is already in a format that can be uploaded, this is a
+    /// no-op.
+    pub fn convert(self: *Image, alloc: Allocator) !void {
+        switch (self.*) {
+            .ready,
+            .unload_pending,
+            .unload_ready,
+            => unreachable, // invalid
+
+            .pending_rgba => {}, // ready
+
+            // RGB needs to be converted to RGBA because Metal textures
+            // don't support RGB.
+            .pending_rgb => |*p| {
+                // Note: this is the slowest possible way to do this...
+                const data = p.dataSlice(3);
+                const pixels = data.len / 3;
+                var rgba = try alloc.alloc(u8, pixels * 4);
+                errdefer alloc.free(rgba);
+                var i: usize = 0;
+                while (i < pixels) : (i += 1) {
+                    const data_i = i * 3;
+                    const rgba_i = i * 4;
+                    rgba[rgba_i] = data[data_i];
+                    rgba[rgba_i + 1] = data[data_i + 1];
+                    rgba[rgba_i + 2] = data[data_i + 2];
+                    rgba[rgba_i + 3] = 255;
+                }
+
+                alloc.free(data);
+                p.data = rgba.ptr;
+                self.* = .{ .pending_rgba = p.* };
+            },
+        }
+    }
+
+    /// Upload the pending image to the GPU and change the state of this
+    /// image to ready.
+    pub fn upload(
+        self: *Image,
+        alloc: Allocator,
+    ) !void {
+        // Convert our data if we have to
+        try self.convert(alloc);
+
+        // Get our pending info
+        const p = self.pending().?;
+
+        // Get our format
+        const formats: struct {
+            internal: gl.Texture.InternalFormat,
+            format: gl.Texture.Format,
+        } = switch (self.*) {
+            .pending_rgb => .{ .internal = .rgb, .format = .rgb },
+            .pending_rgba => .{ .internal = .rgba, .format = .rgba },
+            else => unreachable,
+        };
+
+        // Create our texture
+        const tex = try gl.Texture.create();
+        errdefer tex.destroy();
+
+        const texbind = try tex.bind(.@"2D");
+        try texbind.parameter(.WrapS, gl.c.GL_CLAMP_TO_EDGE);
+        try texbind.parameter(.WrapT, gl.c.GL_CLAMP_TO_EDGE);
+        try texbind.parameter(.MinFilter, gl.c.GL_LINEAR);
+        try texbind.parameter(.MagFilter, gl.c.GL_LINEAR);
+        try texbind.image2D(
+            0,
+            formats.internal,
+            @intCast(p.width),
+            @intCast(p.height),
+            0,
+            formats.format,
+            .UnsignedByte,
+            p.data,
+        );
+
+        // Uploaded. We can now clear our data and change our state.
+        self.deinit(alloc);
+        self.* = .{ .ready = tex };
+    }
+
+    /// Our pixel depth
+    fn depth(self: Image) u32 {
+        return switch (self) {
+            .pending_rgb => 3,
+            .pending_rgba => 4,
+            else => unreachable,
+        };
+    }
+
+    /// Returns true if this image is in a pending state and requires upload.
+    fn pending(self: Image) ?Pending {
+        return switch (self) {
+            .pending_rgb,
+            .pending_rgba,
+            => |p| p,
+
+            else => null,
+        };
+    }
+};

From 64cacce1cf6f8f3129c294601a0122bbe1f0f1ad Mon Sep 17 00:00:00 2001
From: Mitchell Hashimoto <mitchell.hashimoto@gmail.com>
Date: Sun, 19 Nov 2023 22:33:06 -0800
Subject: [PATCH 3/7] renderer/opengl: setup image uniforms

---
 src/renderer/OpenGL.zig              |  55 +++++++----
 src/renderer/opengl/ImageProgram.zig | 134 +++++++++++++++++++++++++++
 src/renderer/shaders/image.f.glsl    |  11 +++
 src/renderer/shaders/image.v.glsl    |  44 +++++++++
 4 files changed, 227 insertions(+), 17 deletions(-)
 create mode 100644 src/renderer/opengl/ImageProgram.zig
 create mode 100644 src/renderer/shaders/image.f.glsl
 create mode 100644 src/renderer/shaders/image.v.glsl

diff --git a/src/renderer/OpenGL.zig b/src/renderer/OpenGL.zig
index ab1ee27d0..bcd8c8046 100644
--- a/src/renderer/OpenGL.zig
+++ b/src/renderer/OpenGL.zig
@@ -22,6 +22,7 @@ const math = @import("../math.zig");
 const Surface = @import("../Surface.zig");
 
 const CellProgram = @import("opengl/CellProgram.zig");
+const ImageProgram = @import("opengl/ImageProgram.zig");
 const gl_image = @import("opengl/image.zig");
 const custom = @import("opengl/custom.zig");
 const Image = gl_image.Image;
@@ -148,17 +149,22 @@ const SetScreenSize = struct {
         );
 
         // Update the projection uniform within our shader
-        try gl_state.cell_program.program.setUniform(
-            "projection",
+        inline for (.{ "cell_program", "image_program" }) |name| {
+            const program = @field(gl_state, name);
+            const bind = try program.program.use();
+            defer bind.unbind();
+            try program.program.setUniform(
+                "projection",
 
-            // 2D orthographic projection with the full w/h
-            math.ortho2d(
-                -1 * @as(f32, @floatFromInt(padding.left)),
-                @floatFromInt(padded_size.width + padding.right),
-                @floatFromInt(padded_size.height + padding.bottom),
-                -1 * @as(f32, @floatFromInt(padding.top)),
-            ),
-        );
+                // 2D orthographic projection with the full w/h
+                math.ortho2d(
+                    -1 * @as(f32, @floatFromInt(padding.left)),
+                    @floatFromInt(padded_size.width + padding.right),
+                    @floatFromInt(padded_size.height + padding.bottom),
+                    -1 * @as(f32, @floatFromInt(padding.top)),
+                ),
+            );
+        }
 
         // Update our custom shader resolution
         if (gl_state.custom) |*custom_state| {
@@ -173,13 +179,21 @@ const SetFontSize = struct {
     fn apply(self: SetFontSize, r: *const OpenGL) !void {
         const gl_state = r.gl_state orelse return error.OpenGLUninitialized;
 
-        try gl_state.cell_program.program.setUniform(
-            "cell_size",
-            @Vector(2, f32){
-                @floatFromInt(self.metrics.cell_width),
-                @floatFromInt(self.metrics.cell_height),
-            },
-        );
+        inline for (.{ "cell_program", "image_program" }) |name| {
+            const program = @field(gl_state, name);
+            const bind = try program.program.use();
+            defer bind.unbind();
+            try program.program.setUniform(
+                "cell_size",
+                @Vector(2, f32){
+                    @floatFromInt(self.metrics.cell_width),
+                    @floatFromInt(self.metrics.cell_height),
+                },
+            );
+        }
+
+        const bind = try gl_state.cell_program.program.use();
+        defer bind.unbind();
         try gl_state.cell_program.program.setUniform(
             "strikethrough_position",
             @as(f32, @floatFromInt(self.metrics.strikethrough_position)),
@@ -1743,6 +1757,7 @@ fn drawCells(
 /// OpenGL context is replaced.
 const GLState = struct {
     cell_program: CellProgram,
+    image_program: ImageProgram,
     texture: gl.Texture,
     texture_color: gl.Texture,
     custom: ?custom.State,
@@ -1830,8 +1845,13 @@ const GLState = struct {
         const cell_program = try CellProgram.init();
         errdefer cell_program.deinit();
 
+        // Build our image renderer
+        const image_program = try ImageProgram.init();
+        errdefer image_program.deinit();
+
         return .{
             .cell_program = cell_program,
+            .image_program = image_program,
             .texture = tex,
             .texture_color = tex_color,
             .custom = custom_state,
@@ -1842,6 +1862,7 @@ const GLState = struct {
         if (self.custom) |v| v.deinit(alloc);
         self.texture.destroy();
         self.texture_color.destroy();
+        self.image_program.deinit();
         self.cell_program.deinit();
     }
 };
diff --git a/src/renderer/opengl/ImageProgram.zig b/src/renderer/opengl/ImageProgram.zig
new file mode 100644
index 000000000..e53891818
--- /dev/null
+++ b/src/renderer/opengl/ImageProgram.zig
@@ -0,0 +1,134 @@
+/// The OpenGL program for rendering terminal cells.
+const ImageProgram = @This();
+
+const std = @import("std");
+const gl = @import("opengl");
+
+program: gl.Program,
+vao: gl.VertexArray,
+ebo: gl.Buffer,
+vbo: gl.Buffer,
+
+pub const Input = extern struct {
+    /// vec2 grid_coord
+    grid_col: u16,
+    grid_row: u16,
+
+    /// vec2 cell_offset
+    cell_offset_x: u32 = 0,
+    cell_offset_y: u32 = 0,
+
+    /// vec4 source_rect
+    source_x: u32 = 0,
+    source_y: u32 = 0,
+    source_width: u32 = 0,
+    source_height: u32 = 0,
+
+    /// vec2 dest_size
+    dest_width: u32 = 0,
+    dest_height: u32 = 0,
+};
+
+pub fn init() !ImageProgram {
+    // Load and compile our shaders.
+    const program = try gl.Program.createVF(
+        @embedFile("../shaders/image.v.glsl"),
+        @embedFile("../shaders/image.f.glsl"),
+    );
+    errdefer program.destroy();
+
+    // Set our program uniforms
+    const pbind = try program.use();
+    defer pbind.unbind();
+
+    // Set all of our texture indexes
+    try program.setUniform("image", 0);
+
+    // Setup our VAO
+    const vao = try gl.VertexArray.create();
+    errdefer vao.destroy();
+    const vaobind = try vao.bind();
+    defer vaobind.unbind();
+
+    // Element buffer (EBO)
+    const ebo = try gl.Buffer.create();
+    errdefer ebo.destroy();
+    var ebobind = try ebo.bind(.element_array);
+    defer ebobind.unbind();
+    try ebobind.setData([6]u8{
+        0, 1, 3, // Top-left triangle
+        1, 2, 3, // Bottom-right triangle
+    }, .static_draw);
+
+    // Vertex buffer (VBO)
+    const vbo = try gl.Buffer.create();
+    errdefer vbo.destroy();
+    var vbobind = try vbo.bind(.array);
+    defer vbobind.unbind();
+    var offset: usize = 0;
+    try vbobind.attributeAdvanced(0, 2, gl.c.GL_UNSIGNED_SHORT, false, @sizeOf(Input), offset);
+    offset += 2 * @sizeOf(u16);
+    try vbobind.attributeAdvanced(1, 2, gl.c.GL_UNSIGNED_INT, false, @sizeOf(Input), offset);
+    offset += 2 * @sizeOf(u32);
+    try vbobind.attributeAdvanced(2, 4, gl.c.GL_UNSIGNED_INT, false, @sizeOf(Input), offset);
+    offset += 4 * @sizeOf(u32);
+    try vbobind.attributeAdvanced(3, 2, gl.c.GL_UNSIGNED_INT, false, @sizeOf(Input), offset);
+    offset += 2 * @sizeOf(u32);
+    try vbobind.enableAttribArray(0);
+    try vbobind.enableAttribArray(1);
+    try vbobind.enableAttribArray(2);
+    try vbobind.enableAttribArray(3);
+    try vbobind.attributeDivisor(0, 1);
+    try vbobind.attributeDivisor(1, 1);
+    try vbobind.attributeDivisor(2, 1);
+    try vbobind.attributeDivisor(3, 1);
+
+    return .{
+        .program = program,
+        .vao = vao,
+        .ebo = ebo,
+        .vbo = vbo,
+    };
+}
+
+pub fn bind(self: ImageProgram) !Binding {
+    const program = try self.program.use();
+    errdefer program.unbind();
+
+    const vao = try self.vao.bind();
+    errdefer vao.unbind();
+
+    const ebo = try self.ebo.bind(.element_array);
+    errdefer ebo.unbind();
+
+    const vbo = try self.vbo.bind(.array);
+    errdefer vbo.unbind();
+
+    return .{
+        .program = program,
+        .vao = vao,
+        .ebo = ebo,
+        .vbo = vbo,
+    };
+}
+
+pub fn deinit(self: ImageProgram) void {
+    self.vbo.destroy();
+    self.ebo.destroy();
+    self.vao.destroy();
+    self.program.destroy();
+}
+
+pub const Binding = struct {
+    program: gl.Program.Binding,
+    vao: gl.VertexArray.Binding,
+    ebo: gl.Buffer.Binding,
+    vbo: gl.Buffer.Binding,
+
+    pub fn unbind(self: Binding) void {
+        self.vbo.unbind();
+        self.ebo.unbind();
+        self.vao.unbind();
+        self.program.unbind();
+    }
+};
diff --git a/src/renderer/shaders/image.f.glsl b/src/renderer/shaders/image.f.glsl
new file mode 100644
index 000000000..9373abe11
--- /dev/null
+++ b/src/renderer/shaders/image.f.glsl
@@ -0,0 +1,11 @@
+#version 330 core
+
+in vec2 tex_coords;
+
+layout(location = 0) out vec4 out_FragColor;
+
+uniform sampler2D image;
+
+void main() {
+    out_FragColor = texture(image, tex_coords);
+}
diff --git a/src/renderer/shaders/image.v.glsl b/src/renderer/shaders/image.v.glsl
new file mode 100644
index 000000000..e3d07ca9e
--- /dev/null
+++ b/src/renderer/shaders/image.v.glsl
@@ -0,0 +1,44 @@
+#version 330 core
+
+layout (location = 0) in vec2 grid_pos;
+layout (location = 1) in vec2 cell_offset;
+layout (location = 2) in vec4 source_rect;
+layout (location = 3) in vec2 dest_size;
+
+out vec2 tex_coord;
+
+uniform sampler2D image;
+uniform vec2 cell_size;
+uniform mat4 projection;
+
+void main() {
+    // The size of the image in pixels
+    vec2 image_size = textureSize(image, 0);
+
+    // Turn the cell position into a vertex point depending on the
+    // gl_VertexID. Since we use instanced drawing, we have 4 vertices
+    // for each corner of the cell. We can use gl_VertexID to determine
+    // which one we're looking at. Using this, we can use 1 or 0 to keep
+    // or discard the value for the vertex.
+    //
+    // 0 = top-right
+    // 1 = bot-right
+    // 2 = bot-left
+    // 3 = top-left
+    vec2 position;
+    position.x = (gl_VertexID == 0 || gl_VertexID == 1) ? 1. : 0.;
+    position.y = (gl_VertexID == 0 || gl_VertexID == 3) ? 0. : 1.;
+
+    // The texture coordinates start at our source x/y, then add the width/height
+    // as enabled by our instance id, then normalize to [0, 1]
+    tex_coord = source_rect.xy;
+    tex_coord += source_rect.zw * position;
+    tex_coord /= image_size;
+
+    // The position of our image starts at the top-left of the grid cell and
+    // adds the source rect width/height components.
+    vec2 image_pos = (cell_size * grid_pos) + cell_offset;
+    image_pos += dest_size * position;
+
+    gl_Position = projection * vec4(image_pos.xy, 0, 1.0);
+}

From 2a10af90a3113c008766cbe52d41d91736100709 Mon Sep 17 00:00:00 2001
From: Mitchell Hashimoto <mitchell.hashimoto@gmail.com>
Date: Sun, 19 Nov 2023 22:43:39 -0800
Subject: [PATCH 4/7] renderer/opengl: draw images

---
 src/renderer/OpenGL.zig | 133 ++++++++++++++++++++++++++++++----------
 1 file changed, 100 insertions(+), 33 deletions(-)

diff --git a/src/renderer/OpenGL.zig b/src/renderer/OpenGL.zig
index bcd8c8046..eb9fde944 100644
--- a/src/renderer/OpenGL.zig
+++ b/src/renderer/OpenGL.zig
@@ -1672,6 +1672,104 @@ fn drawCellProgram(
     );
     gl.clear(gl.c.GL_COLOR_BUFFER_BIT);
 
+    // If we have deferred operations, run them.
+    if (self.deferred_screen_size) |v| {
+        try v.apply(self);
+        self.deferred_screen_size = null;
+    }
+    if (self.deferred_font_size) |v| {
+        try v.apply(self);
+        self.deferred_font_size = null;
+    }
+
+    // Draw background images first
+    try self.drawImages(gl_state, self.image_placements.items[0..self.image_bg_end]);
+
+    // Draw our background
+    try self.drawCells(gl_state, self.cells_bg);
+
+    // Then draw images under text
+    try self.drawImages(gl_state, self.image_placements.items[self.image_bg_end..self.image_text_end]);
+
+    // Drag foreground
+    try self.drawCells(gl_state, self.cells);
+
+    // Draw remaining images
+    try self.drawImages(gl_state, self.image_placements.items[self.image_text_end..]);
+}
+
+/// Runs the image program to draw images.
+fn drawImages(
+    self: *OpenGL,
+    gl_state: *const GLState,
+    placements: []const gl_image.Placement,
+) !void {
+    if (placements.len == 0) return;
+
+    // Bind our image program
+    const bind = try gl_state.image_program.bind();
+    defer bind.unbind();
+
+    // For each placement we need to bind the texture
+    for (placements) |p| {
+        // Get the image and image texture
+        const image = self.images.get(p.image_id) orelse {
+            log.warn("image not found for placement image_id={}", .{p.image_id});
+            continue;
+        };
+
+        const texture = switch (image) {
+            .ready => |t| t,
+            else => {
+                log.warn("image not ready for placement image_id={}", .{p.image_id});
+                continue;
+            },
+        };
+
+        // Bind the texture
+        try gl.Texture.active(gl.c.GL_TEXTURE0);
+        var texbind = try texture.bind(.@"2D");
+        defer texbind.unbind();
+
+        // Setup our data
+        try bind.vbo.setData(ImageProgram.Input{
+            .grid_col = @intCast(p.x),
+            .grid_row = @intCast(p.y),
+            .cell_offset_x = p.cell_offset_x,
+            .cell_offset_y = p.cell_offset_y,
+            .source_x = p.source_x,
+            .source_y = p.source_y,
+            .source_width = p.source_width,
+            .source_height = p.source_height,
+            .dest_width = p.width,
+            .dest_height = p.height,
+        }, .static_draw);
+
+        try gl.drawElementsInstanced(
+            gl.c.GL_TRIANGLES,
+            6,
+            gl.c.GL_UNSIGNED_BYTE,
+            1,
+        );
+    }
+}
+
+/// Loads some set of cell data into our buffer and issues a draw call.
+/// This expects all the OpenGL state to be setup.
+///
+/// Future: when we move to multiple shaders, this will go away and
+/// we'll have a draw call per-shader.
+fn drawCells(
+    self: *OpenGL,
+    gl_state: *const GLState,
+    cells: std.ArrayListUnmanaged(CellProgram.Cell),
+) !void {
+    // If we have no cells to render, then we render nothing.
+    if (cells.items.len == 0) return;
+
+    // Todo: get rid of this completely
+    self.gl_cells_written = 0;
+
     // Bind our cell program state, buffers
     const bind = try gl_state.cell_program.bind();
     defer bind.unbind();
@@ -1685,37 +1783,6 @@ fn drawCellProgram(
     var texbind1 = try gl_state.texture_color.bind(.@"2D");
     defer texbind1.unbind();
 
-    // If we have deferred operations, run them.
-    if (self.deferred_screen_size) |v| {
-        try v.apply(self);
-        self.deferred_screen_size = null;
-    }
-    if (self.deferred_font_size) |v| {
-        try v.apply(self);
-        self.deferred_font_size = null;
-    }
-
-    // Draw our background, then draw the fg on top of it.
-    try self.drawCells(bind.vbo, self.cells_bg);
-    try self.drawCells(bind.vbo, self.cells);
-}
-
-/// Loads some set of cell data into our buffer and issues a draw call.
-/// This expects all the OpenGL state to be setup.
-///
-/// Future: when we move to multiple shaders, this will go away and
-/// we'll have a draw call per-shader.
-fn drawCells(
-    self: *OpenGL,
-    binding: gl.Buffer.Binding,
-    cells: std.ArrayListUnmanaged(CellProgram.Cell),
-) !void {
-    // If we have no cells to render, then we render nothing.
-    if (cells.items.len == 0) return;
-
-    // Todo: get rid of this completely
-    self.gl_cells_written = 0;
-
     // Our allocated buffer on the GPU is smaller than our capacity.
     // We reallocate a new buffer with the full new capacity.
     if (self.gl_cells_size < cells.capacity) {
@@ -1724,7 +1791,7 @@ fn drawCells(
             cells.capacity,
         });
 
-        try binding.setDataNullManual(
+        try bind.vbo.setDataNullManual(
             @sizeOf(CellProgram.Cell) * cells.capacity,
             .static_draw,
         );
@@ -1737,7 +1804,7 @@ fn drawCells(
     if (self.gl_cells_written < cells.items.len) {
         const data = cells.items[self.gl_cells_written..];
         // log.info("sending {} cells to GPU", .{data.len});
-        try binding.setSubData(self.gl_cells_written * @sizeOf(CellProgram.Cell), data);
+        try bind.vbo.setSubData(self.gl_cells_written * @sizeOf(CellProgram.Cell), data);
 
         self.gl_cells_written += data.len;
         assert(data.len > 0);

From a5d71723d540590320a4c8bd4c19cf5a28b21caf Mon Sep 17 00:00:00 2001
From: Mitchell Hashimoto <mitchell.hashimoto@gmail.com>
Date: Sun, 19 Nov 2023 22:49:30 -0800
Subject: [PATCH 5/7] renderer/opengl: do not need to convert

---
 src/renderer/opengl/image.zig | 40 -----------------------------------
 1 file changed, 40 deletions(-)

diff --git a/src/renderer/opengl/image.zig b/src/renderer/opengl/image.zig
index 918149e93..a10fe9600 100644
--- a/src/renderer/opengl/image.zig
+++ b/src/renderer/opengl/image.zig
@@ -114,52 +114,12 @@ pub const Image = union(enum) {
         };
     }
 
-    /// Converts the image data to a format that can be uploaded to the GPU.
-    /// If the data is already in a format that can be uploaded, this is a
-    /// no-op.
-    pub fn convert(self: *Image, alloc: Allocator) !void {
-        switch (self.*) {
-            .ready,
-            .unload_pending,
-            .unload_ready,
-            => unreachable, // invalid
-
-            .pending_rgba => {}, // ready
-
-            // RGB needs to be converted to RGBA because Metal textures
-            // don't support RGB.
-            .pending_rgb => |*p| {
-                // Note: this is the slowest possible way to do this...
-                const data = p.dataSlice(3);
-                const pixels = data.len / 3;
-                var rgba = try alloc.alloc(u8, pixels * 4);
-                errdefer alloc.free(rgba);
-                var i: usize = 0;
-                while (i < pixels) : (i += 1) {
-                    const data_i = i * 3;
-                    const rgba_i = i * 4;
-                    rgba[rgba_i] = data[data_i];
-                    rgba[rgba_i + 1] = data[data_i + 1];
-                    rgba[rgba_i + 2] = data[data_i + 2];
-                    rgba[rgba_i + 3] = 255;
-                }
-
-                alloc.free(data);
-                p.data = rgba.ptr;
-                self.* = .{ .pending_rgba = p.* };
-            },
-        }
-    }
-
     /// Upload the pending image to the GPU and change the state of this
     /// image to ready.
     pub fn upload(
         self: *Image,
         alloc: Allocator,
     ) !void {
-        // Convert our data if we have to
-        try self.convert(alloc);
-
         // Get our pending info
         const p = self.pending().?;
 

From d1de53ed22b33a48a55765310371c659a919efbd Mon Sep 17 00:00:00 2001
From: Mitchell Hashimoto <mitchell.hashimoto@gmail.com>
Date: Sun, 19 Nov 2023 22:52:40 -0800
Subject: [PATCH 6/7] renderer/opengl: correct shader params

---
 src/renderer/shaders/image.f.glsl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/renderer/shaders/image.f.glsl b/src/renderer/shaders/image.f.glsl
index 9373abe11..c9623b6f9 100644
--- a/src/renderer/shaders/image.f.glsl
+++ b/src/renderer/shaders/image.f.glsl
@@ -1,11 +1,11 @@
 #version 330 core
 
-in vec2 tex_coords;
+in vec2 tex_coord;
 
 layout(location = 0) out vec4 out_FragColor;
 
 uniform sampler2D image;
 
 void main() {
-    out_FragColor = texture(image, tex_coords);
+    out_FragColor = texture(image, tex_coord);
 }

From 9988dedb80786e863129d411f7dd22f885eb3f6f Mon Sep 17 00:00:00 2001
From: Mitchell Hashimoto <mitchell.hashimoto@gmail.com>
Date: Mon, 20 Nov 2023 09:45:38 -0800
Subject: [PATCH 7/7] renderr/opengl: stylistic

---
 src/renderer/Metal.zig  |  2 +-
 src/renderer/OpenGL.zig | 15 ++++++++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/renderer/Metal.zig b/src/renderer/Metal.zig
index 1012d6bea..b22085d2a 100644
--- a/src/renderer/Metal.zig
+++ b/src/renderer/Metal.zig
@@ -755,7 +755,7 @@ pub fn drawFrame(self: *Metal, surface: *apprt.Surface) !void {
         try self.drawCells(encoder, &self.buf_cells_bg, self.cells_bg);
 
         // Then draw images under text
-        try self.drawImagePlacements(encoder, self.image_placements.items[0..self.image_text_end]);
+        try self.drawImagePlacements(encoder, self.image_placements.items[self.image_bg_end..self.image_text_end]);
 
         // Then draw fg cells
         try self.drawCells(encoder, &self.buf_cells, self.cells);
diff --git a/src/renderer/OpenGL.zig b/src/renderer/OpenGL.zig
index eb9fde944..7154bbff7 100644
--- a/src/renderer/OpenGL.zig
+++ b/src/renderer/OpenGL.zig
@@ -1683,19 +1683,28 @@ fn drawCellProgram(
     }
 
     // Draw background images first
-    try self.drawImages(gl_state, self.image_placements.items[0..self.image_bg_end]);
+    try self.drawImages(
+        gl_state,
+        self.image_placements.items[0..self.image_bg_end],
+    );
 
     // Draw our background
     try self.drawCells(gl_state, self.cells_bg);
 
     // Then draw images under text
-    try self.drawImages(gl_state, self.image_placements.items[self.image_bg_end..self.image_text_end]);
+    try self.drawImages(
+        gl_state,
+        self.image_placements.items[self.image_bg_end..self.image_text_end],
+    );
 
     // Drag foreground
     try self.drawCells(gl_state, self.cells);
 
     // Draw remaining images
-    try self.drawImages(gl_state, self.image_placements.items[self.image_text_end..]);
+    try self.drawImages(
+        gl_state,
+        self.image_placements.items[self.image_text_end..],
+    );
 }
 
 /// Runs the image program to draw images.