renderer/opengl: upload kitty image textures

pkg/opengl/Texture.zig

@@ -78,6 +78,7 @@ pub const InternalFormat = enum(c_int) {
 pub const Format = enum(c_uint) {
     red = c.GL_RED,
     rgb = c.GL_RGB,
+    rgba = c.GL_RGBA,
     bgra = c.GL_BGRA,
 
     // There are so many more that I haven't filled in.

src/renderer/OpenGL.zig

@@ -22,7 +22,11 @@ const math = @import("../math.zig");
 const Surface = @import("../Surface.zig");
 
 const CellProgram = @import("opengl/CellProgram.zig");
+const gl_image = @import("opengl/image.zig");
 const custom = @import("opengl/custom.zig");
+const Image = gl_image.Image;
+const ImageMap = gl_image.ImageMap;
+const ImagePlacementList = std.ArrayListUnmanaged(gl_image.Placement);
 
 const log = std.log.scoped(.grid);
 
@@ -103,6 +107,12 @@ draw_mutex: DrawMutex = drawMutexZero,
 /// terminal is in reversed mode.
 draw_background: terminal.color.RGB,
 
+/// The images that we may render.
+images: ImageMap = .{},
+image_placements: ImagePlacementList = .{},
+image_bg_end: u32 = 0,
+image_text_end: u32 = 0,
+
 /// Defererred OpenGL operation to update the screen size.
 const SetScreenSize = struct {
     size: renderer.ScreenSize,
@@ -307,6 +317,13 @@ pub fn init(alloc: Allocator, options: renderer.Options) !OpenGL {
 pub fn deinit(self: *OpenGL) void {
     self.font_shaper.deinit();
 
+    {
+        var it = self.images.iterator();
+        while (it.next()) |kv| kv.value_ptr.deinit(self.alloc);
+        self.images.deinit(self.alloc);
+    }
+    self.image_placements.deinit(self.alloc);
+
     if (self.gl_state) |*v| v.deinit(self.alloc);
 
     self.cells.deinit(self.alloc);
@@ -623,6 +640,13 @@ pub fn updateFrame(
             cursor_blink_visible,
         );
 
+        // If we have Kitty graphics data, we enter a SLOW SLOW SLOW path.
+        // We only do this if the Kitty image state is dirty meaning only if
+        // it changes.
+        if (state.terminal.screen.kitty_images.dirty) {
+            try self.prepKittyGraphics(state.terminal);
+        }
+
         break :critical .{
             .gl_bg = self.background_color,
             .selection = selection,
@@ -651,6 +675,158 @@ pub fn updateFrame(
     }
 }
 
+/// This goes through the Kitty graphic placements and accumulates the
+/// placements we need to render on our viewport. It also ensures that
+/// the visible images are loaded on the GPU.
+fn prepKittyGraphics(
+    self: *OpenGL,
+    t: *terminal.Terminal,
+) !void {
+    const storage = &t.screen.kitty_images;
+    defer storage.dirty = false;
+
+    // We always clear our previous placements no matter what because
+    // we rebuild them from scratch.
+    self.image_placements.clearRetainingCapacity();
+
+    // Go through our known images and if there are any that are no longer
+    // in use then mark them to be freed.
+    //
+    // This never conflicts with the below because a placement can't
+    // reference an image that doesn't exist.
+    {
+        var it = self.images.iterator();
+        while (it.next()) |kv| {
+            if (storage.imageById(kv.key_ptr.*) == null) {
+                kv.value_ptr.markForUnload();
+            }
+        }
+    }
+
+    // The top-left and bottom-right corners of our viewport in screen
+    // points. This lets us determine offsets and containment of placements.
+    const top = (terminal.point.Viewport{}).toScreen(&t.screen);
+    const bot = (terminal.point.Viewport{
+        .x = t.screen.cols - 1,
+        .y = t.screen.rows - 1,
+    }).toScreen(&t.screen);
+
+    // Go through the placements and ensure the image is loaded on the GPU.
+    var it = storage.placements.iterator();
+    while (it.next()) |kv| {
+        // Find the image in storage
+        const p = kv.value_ptr;
+        const image = storage.imageById(kv.key_ptr.image_id) orelse {
+            log.warn(
+                "missing image for placement, ignoring image_id={}",
+                .{kv.key_ptr.image_id},
+            );
+            continue;
+        };
+
+        // If the selection isn't within our viewport then skip it.
+        const rect = p.rect(image, t);
+        if (rect.top_left.y > bot.y) continue;
+        if (rect.bottom_right.y < top.y) continue;
+
+        // If the top left is outside the viewport we need to calc an offset
+        // so that we render (0, 0) with some offset for the texture.
+        const offset_y: u32 = if (rect.top_left.y < t.screen.viewport) offset_y: {
+            const offset_cells = t.screen.viewport - rect.top_left.y;
+            const offset_pixels = offset_cells * self.cell_size.height;
+            break :offset_y @intCast(offset_pixels);
+        } else 0;
+
+        // If we already know about this image then do nothing
+        const gop = try self.images.getOrPut(self.alloc, kv.key_ptr.image_id);
+        if (!gop.found_existing) {
+            // Copy the data into the pending state.
+            const data = try self.alloc.dupe(u8, image.data);
+            errdefer self.alloc.free(data);
+
+            // Store it in the map
+            const pending: Image.Pending = .{
+                .width = image.width,
+                .height = image.height,
+                .data = data.ptr,
+            };
+
+            gop.value_ptr.* = switch (image.format) {
+                .rgb => .{ .pending_rgb = pending },
+                .rgba => .{ .pending_rgba = pending },
+                .png => unreachable, // should be decoded by now
+            };
+        }
+
+        // Convert our screen point to a viewport point
+        const viewport = p.point.toViewport(&t.screen);
+
+        // Calculate the source rectangle
+        const source_x = @min(image.width, p.source_x);
+        const source_y = @min(image.height, p.source_y + offset_y);
+        const source_width = if (p.source_width > 0)
+            @min(image.width - source_x, p.source_width)
+        else
+            image.width;
+        const source_height = if (p.source_height > 0)
+            @min(image.height, p.source_height)
+        else
+            image.height -| offset_y;
+
+        // Calculate the width/height of our image.
+        const dest_width = if (p.columns > 0) p.columns * self.cell_size.width else source_width;
+        const dest_height = if (p.rows > 0) p.rows * self.cell_size.height else source_height;
+
+        // Accumulate the placement
+        if (image.width > 0 and image.height > 0) {
+            try self.image_placements.append(self.alloc, .{
+                .image_id = kv.key_ptr.image_id,
+                .x = @intCast(p.point.x),
+                .y = @intCast(viewport.y),
+                .z = p.z,
+                .width = dest_width,
+                .height = dest_height,
+                .cell_offset_x = p.x_offset,
+                .cell_offset_y = p.y_offset,
+                .source_x = source_x,
+                .source_y = source_y,
+                .source_width = source_width,
+                .source_height = source_height,
+            });
+        }
+    }
+
+    // Sort the placements by their Z value.
+    std.mem.sortUnstable(
+        gl_image.Placement,
+        self.image_placements.items,
+        {},
+        struct {
+            fn lessThan(
+                ctx: void,
+                lhs: gl_image.Placement,
+                rhs: gl_image.Placement,
+            ) bool {
+                _ = ctx;
+                return lhs.z < rhs.z or (lhs.z == rhs.z and lhs.image_id < rhs.image_id);
+            }
+        }.lessThan,
+    );
+
+    // Find our indices
+    self.image_bg_end = 0;
+    self.image_text_end = 0;
+    const bg_limit = std.math.minInt(i32) / 2;
+    for (self.image_placements.items, 0..) |p, i| {
+        if (self.image_bg_end == 0 and p.z >= bg_limit) {
+            self.image_bg_end = @intCast(i);
+        }
+        if (self.image_text_end == 0 and p.z >= 0) {
+            self.image_text_end = @intCast(i);
+        }
+    }
+}
+
 /// rebuildCells rebuilds all the GPU cells from our CPU state. This is a
 /// slow operation but ensures that the GPU state exactly matches the CPU state.
 /// In steady-state operation, we use some GPU tricks to send down stale data
@@ -1396,6 +1572,27 @@ pub fn drawFrame(self: *OpenGL, surface: *apprt.Surface) !void {
     defer if (single_threaded_draw) self.draw_mutex.unlock();
     const gl_state: *GLState = if (self.gl_state) |*v| v else return;
 
+    // Go through our images and see if we need to setup any textures.
+    {
+        var image_it = self.images.iterator();
+        while (image_it.next()) |kv| {
+            switch (kv.value_ptr.*) {
+                .ready => {},
+
+                .pending_rgb,
+                .pending_rgba,
+                => try kv.value_ptr.upload(self.alloc),
+
+                .unload_pending,
+                .unload_ready,
+                => {
+                    kv.value_ptr.deinit(self.alloc);
+                    self.images.removeByPtr(kv.key_ptr);
+                },
+            }
+        }
+    }
+
     // Draw our terminal cells
     try self.drawCellProgram(gl_state);
 

src/renderer/opengl/image.zig

@@ -0,0 +1,220 @@
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const gl = @import("opengl");
+
+/// Represents a single image placement on the grid. A placement is a
+/// request to render an instance of an image.
+pub const Placement = struct {
+    /// The image being rendered. This MUST be in the image map.
+    image_id: u32,
+
+    /// The grid x/y where this placement is located.
+    x: u32,
+    y: u32,
+    z: i32,
+
+    /// The width/height of the placed image.
+    width: u32,
+    height: u32,
+
+    /// The offset in pixels from the top left of the cell. This is
+    /// clamped to the size of a cell.
+    cell_offset_x: u32,
+    cell_offset_y: u32,
+
+    /// The source rectangle of the placement.
+    source_x: u32,
+    source_y: u32,
+    source_width: u32,
+    source_height: u32,
+};
+
+/// The map used for storing images.
+pub const ImageMap = std.AutoHashMapUnmanaged(u32, Image);
+
+/// The state for a single image that is to be rendered. The image can be
+/// pending upload or ready to use with a texture.
+pub const Image = union(enum) {
+    /// The image is pending upload to the GPU. The different keys are
+    /// different formats since some formats aren't accepted by the GPU
+    /// and require conversion.
+    ///
+    /// This data is owned by this union so it must be freed once the
+    /// image is uploaded.
+    pending_rgb: Pending,
+    pending_rgba: Pending,
+
+    /// The image is uploaded and ready to be used.
+    ready: gl.Texture,
+
+    /// The image is uploaded but is scheduled to be unloaded.
+    unload_pending: []u8,
+    unload_ready: gl.Texture,
+
+    /// Pending image data that needs to be uploaded to the GPU.
+    pub const Pending = struct {
+        height: u32,
+        width: u32,
+
+        /// Data is always expected to be (width * height * depth). Depth
+        /// is based on the union key.
+        data: [*]u8,
+
+        pub fn dataSlice(self: Pending, d: u32) []u8 {
+            return self.data[0..self.len(d)];
+        }
+
+        pub fn len(self: Pending, d: u32) u32 {
+            return self.width * self.height * d;
+        }
+    };
+
+    pub fn deinit(self: Image, alloc: Allocator) void {
+        switch (self) {
+            .pending_rgb => |p| alloc.free(p.dataSlice(3)),
+            .pending_rgba => |p| alloc.free(p.dataSlice(4)),
+            .unload_pending => |data| alloc.free(data),
+
+            .ready,
+            .unload_ready,
+            => |tex| tex.destroy(),
+        }
+    }
+
+    /// Mark this image for unload whatever state it is in.
+    pub fn markForUnload(self: *Image) void {
+        self.* = switch (self.*) {
+            .unload_pending,
+            .unload_ready,
+            => return,
+
+            .ready => |obj| .{ .unload_ready = obj },
+            .pending_rgb => |p| .{ .unload_pending = p.dataSlice(3) },
+            .pending_rgba => |p| .{ .unload_pending = p.dataSlice(4) },
+        };
+    }
+
+    /// Returns true if this image is pending upload.
+    pub fn isPending(self: Image) bool {
+        return self.pending() != null;
+    }
+
+    /// Returns true if this image is pending an unload.
+    pub fn isUnloading(self: Image) bool {
+        return switch (self) {
+            .unload_pending,
+            .unload_ready,
+            => true,
+
+            .ready,
+            .pending_rgb,
+            .pending_rgba,
+            => false,
+        };
+    }
+
+    /// Converts the image data to a format that can be uploaded to the GPU.
+    /// If the data is already in a format that can be uploaded, this is a
+    /// no-op.
+    pub fn convert(self: *Image, alloc: Allocator) !void {
+        switch (self.*) {
+            .ready,
+            .unload_pending,
+            .unload_ready,
+            => unreachable, // invalid
+
+            .pending_rgba => {}, // ready
+
+            // RGB needs to be converted to RGBA because Metal textures
+            // don't support RGB.
+            .pending_rgb => |*p| {
+                // Note: this is the slowest possible way to do this...
+                const data = p.dataSlice(3);
+                const pixels = data.len / 3;
+                var rgba = try alloc.alloc(u8, pixels * 4);
+                errdefer alloc.free(rgba);
+                var i: usize = 0;
+                while (i < pixels) : (i += 1) {
+                    const data_i = i * 3;
+                    const rgba_i = i * 4;
+                    rgba[rgba_i] = data[data_i];
+                    rgba[rgba_i + 1] = data[data_i + 1];
+                    rgba[rgba_i + 2] = data[data_i + 2];
+                    rgba[rgba_i + 3] = 255;
+                }
+
+                alloc.free(data);
+                p.data = rgba.ptr;
+                self.* = .{ .pending_rgba = p.* };
+            },
+        }
+    }
+
+    /// Upload the pending image to the GPU and change the state of this
+    /// image to ready.
+    pub fn upload(
+        self: *Image,
+        alloc: Allocator,
+    ) !void {
+        // Convert our data if we have to
+        try self.convert(alloc);
+
+        // Get our pending info
+        const p = self.pending().?;
+
+        // Get our format
+        const formats: struct {
+            internal: gl.Texture.InternalFormat,
+            format: gl.Texture.Format,
+        } = switch (self.*) {
+            .pending_rgb => .{ .internal = .rgb, .format = .rgb },
+            .pending_rgba => .{ .internal = .rgba, .format = .rgba },
+            else => unreachable,
+        };
+
+        // Create our texture
+        const tex = try gl.Texture.create();
+        errdefer tex.destroy();
+
+        const texbind = try tex.bind(.@"2D");
+        try texbind.parameter(.WrapS, gl.c.GL_CLAMP_TO_EDGE);
+        try texbind.parameter(.WrapT, gl.c.GL_CLAMP_TO_EDGE);
+        try texbind.parameter(.MinFilter, gl.c.GL_LINEAR);
+        try texbind.parameter(.MagFilter, gl.c.GL_LINEAR);
+        try texbind.image2D(
+            0,
+            formats.internal,
+            @intCast(p.width),
+            @intCast(p.height),
+            0,
+            formats.format,
+            .UnsignedByte,
+            p.data,
+        );
+
+        // Uploaded. We can now clear our data and change our state.
+        self.deinit(alloc);
+        self.* = .{ .ready = tex };
+    }
+
+    /// Our pixel depth
+    fn depth(self: Image) u32 {
+        return switch (self) {
+            .pending_rgb => 3,
+            .pending_rgba => 4,
+            else => unreachable,
+        };
+    }
+
+    /// Returns true if this image is in a pending state and requires upload.
+    fn pending(self: Image) ?Pending {
+        return switch (self) {
+            .pending_rgb,
+            .pending_rgba,
+            => |p| p,
+
+            else => null,
+        };
+    }
+};