From 76c76ce85ef5e675bfd598c153504fc7c7e6746d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 19 Nov 2023 22:08:07 -0800 Subject: [PATCH] renderer/opengl: upload kitty image textures --- pkg/opengl/Texture.zig | 1 + src/renderer/OpenGL.zig | 197 ++++++++++++++++++++++++++++++ src/renderer/opengl/image.zig | 220 ++++++++++++++++++++++++++++++++++ 3 files changed, 418 insertions(+) create mode 100644 src/renderer/opengl/image.zig diff --git a/pkg/opengl/Texture.zig b/pkg/opengl/Texture.zig index afa22e926..87f3ca102 100644 --- a/pkg/opengl/Texture.zig +++ b/pkg/opengl/Texture.zig @@ -78,6 +78,7 @@ pub const InternalFormat = enum(c_int) { pub const Format = enum(c_uint) { red = c.GL_RED, rgb = c.GL_RGB, + rgba = c.GL_RGBA, bgra = c.GL_BGRA, // There are so many more that I haven't filled in. diff --git a/src/renderer/OpenGL.zig b/src/renderer/OpenGL.zig index 3d7031c8d..ab1ee27d0 100644 --- a/src/renderer/OpenGL.zig +++ b/src/renderer/OpenGL.zig @@ -22,7 +22,11 @@ const math = @import("../math.zig"); const Surface = @import("../Surface.zig"); const CellProgram = @import("opengl/CellProgram.zig"); +const gl_image = @import("opengl/image.zig"); const custom = @import("opengl/custom.zig"); +const Image = gl_image.Image; +const ImageMap = gl_image.ImageMap; +const ImagePlacementList = std.ArrayListUnmanaged(gl_image.Placement); const log = std.log.scoped(.grid); @@ -103,6 +107,12 @@ draw_mutex: DrawMutex = drawMutexZero, /// terminal is in reversed mode. draw_background: terminal.color.RGB, +/// The images that we may render. +images: ImageMap = .{}, +image_placements: ImagePlacementList = .{}, +image_bg_end: u32 = 0, +image_text_end: u32 = 0, + /// Defererred OpenGL operation to update the screen size. const SetScreenSize = struct { size: renderer.ScreenSize, @@ -307,6 +317,13 @@ pub fn init(alloc: Allocator, options: renderer.Options) !OpenGL { pub fn deinit(self: *OpenGL) void { self.font_shaper.deinit(); + { + var it = self.images.iterator(); + while (it.next()) |kv| kv.value_ptr.deinit(self.alloc); + self.images.deinit(self.alloc); + } + self.image_placements.deinit(self.alloc); + if (self.gl_state) |*v| v.deinit(self.alloc); self.cells.deinit(self.alloc); @@ -623,6 +640,13 @@ pub fn updateFrame( cursor_blink_visible, ); + // If we have Kitty graphics data, we enter a SLOW SLOW SLOW path. + // We only do this if the Kitty image state is dirty meaning only if + // it changes. + if (state.terminal.screen.kitty_images.dirty) { + try self.prepKittyGraphics(state.terminal); + } + break :critical .{ .gl_bg = self.background_color, .selection = selection, @@ -651,6 +675,158 @@ pub fn updateFrame( } } +/// This goes through the Kitty graphic placements and accumulates the +/// placements we need to render on our viewport. It also ensures that +/// the visible images are loaded on the GPU. +fn prepKittyGraphics( + self: *OpenGL, + t: *terminal.Terminal, +) !void { + const storage = &t.screen.kitty_images; + defer storage.dirty = false; + + // We always clear our previous placements no matter what because + // we rebuild them from scratch. + self.image_placements.clearRetainingCapacity(); + + // Go through our known images and if there are any that are no longer + // in use then mark them to be freed. + // + // This never conflicts with the below because a placement can't + // reference an image that doesn't exist. + { + var it = self.images.iterator(); + while (it.next()) |kv| { + if (storage.imageById(kv.key_ptr.*) == null) { + kv.value_ptr.markForUnload(); + } + } + } + + // The top-left and bottom-right corners of our viewport in screen + // points. This lets us determine offsets and containment of placements. + const top = (terminal.point.Viewport{}).toScreen(&t.screen); + const bot = (terminal.point.Viewport{ + .x = t.screen.cols - 1, + .y = t.screen.rows - 1, + }).toScreen(&t.screen); + + // Go through the placements and ensure the image is loaded on the GPU. + var it = storage.placements.iterator(); + while (it.next()) |kv| { + // Find the image in storage + const p = kv.value_ptr; + const image = storage.imageById(kv.key_ptr.image_id) orelse { + log.warn( + "missing image for placement, ignoring image_id={}", + .{kv.key_ptr.image_id}, + ); + continue; + }; + + // If the selection isn't within our viewport then skip it. + const rect = p.rect(image, t); + if (rect.top_left.y > bot.y) continue; + if (rect.bottom_right.y < top.y) continue; + + // If the top left is outside the viewport we need to calc an offset + // so that we render (0, 0) with some offset for the texture. + const offset_y: u32 = if (rect.top_left.y < t.screen.viewport) offset_y: { + const offset_cells = t.screen.viewport - rect.top_left.y; + const offset_pixels = offset_cells * self.cell_size.height; + break :offset_y @intCast(offset_pixels); + } else 0; + + // If we already know about this image then do nothing + const gop = try self.images.getOrPut(self.alloc, kv.key_ptr.image_id); + if (!gop.found_existing) { + // Copy the data into the pending state. + const data = try self.alloc.dupe(u8, image.data); + errdefer self.alloc.free(data); + + // Store it in the map + const pending: Image.Pending = .{ + .width = image.width, + .height = image.height, + .data = data.ptr, + }; + + gop.value_ptr.* = switch (image.format) { + .rgb => .{ .pending_rgb = pending }, + .rgba => .{ .pending_rgba = pending }, + .png => unreachable, // should be decoded by now + }; + } + + // Convert our screen point to a viewport point + const viewport = p.point.toViewport(&t.screen); + + // Calculate the source rectangle + const source_x = @min(image.width, p.source_x); + const source_y = @min(image.height, p.source_y + offset_y); + const source_width = if (p.source_width > 0) + @min(image.width - source_x, p.source_width) + else + image.width; + const source_height = if (p.source_height > 0) + @min(image.height, p.source_height) + else + image.height -| offset_y; + + // Calculate the width/height of our image. + const dest_width = if (p.columns > 0) p.columns * self.cell_size.width else source_width; + const dest_height = if (p.rows > 0) p.rows * self.cell_size.height else source_height; + + // Accumulate the placement + if (image.width > 0 and image.height > 0) { + try self.image_placements.append(self.alloc, .{ + .image_id = kv.key_ptr.image_id, + .x = @intCast(p.point.x), + .y = @intCast(viewport.y), + .z = p.z, + .width = dest_width, + .height = dest_height, + .cell_offset_x = p.x_offset, + .cell_offset_y = p.y_offset, + .source_x = source_x, + .source_y = source_y, + .source_width = source_width, + .source_height = source_height, + }); + } + } + + // Sort the placements by their Z value. + std.mem.sortUnstable( + gl_image.Placement, + self.image_placements.items, + {}, + struct { + fn lessThan( + ctx: void, + lhs: gl_image.Placement, + rhs: gl_image.Placement, + ) bool { + _ = ctx; + return lhs.z < rhs.z or (lhs.z == rhs.z and lhs.image_id < rhs.image_id); + } + }.lessThan, + ); + + // Find our indices + self.image_bg_end = 0; + self.image_text_end = 0; + const bg_limit = std.math.minInt(i32) / 2; + for (self.image_placements.items, 0..) |p, i| { + if (self.image_bg_end == 0 and p.z >= bg_limit) { + self.image_bg_end = @intCast(i); + } + if (self.image_text_end == 0 and p.z >= 0) { + self.image_text_end = @intCast(i); + } + } +} + /// rebuildCells rebuilds all the GPU cells from our CPU state. This is a /// slow operation but ensures that the GPU state exactly matches the CPU state. /// In steady-state operation, we use some GPU tricks to send down stale data @@ -1396,6 +1572,27 @@ pub fn drawFrame(self: *OpenGL, surface: *apprt.Surface) !void { defer if (single_threaded_draw) self.draw_mutex.unlock(); const gl_state: *GLState = if (self.gl_state) |*v| v else return; + // Go through our images and see if we need to setup any textures. + { + var image_it = self.images.iterator(); + while (image_it.next()) |kv| { + switch (kv.value_ptr.*) { + .ready => {}, + + .pending_rgb, + .pending_rgba, + => try kv.value_ptr.upload(self.alloc), + + .unload_pending, + .unload_ready, + => { + kv.value_ptr.deinit(self.alloc); + self.images.removeByPtr(kv.key_ptr); + }, + } + } + } + // Draw our terminal cells try self.drawCellProgram(gl_state); diff --git a/src/renderer/opengl/image.zig b/src/renderer/opengl/image.zig new file mode 100644 index 000000000..918149e93 --- /dev/null +++ b/src/renderer/opengl/image.zig @@ -0,0 +1,220 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const gl = @import("opengl"); + +/// Represents a single image placement on the grid. A placement is a +/// request to render an instance of an image. +pub const Placement = struct { + /// The image being rendered. This MUST be in the image map. + image_id: u32, + + /// The grid x/y where this placement is located. + x: u32, + y: u32, + z: i32, + + /// The width/height of the placed image. + width: u32, + height: u32, + + /// The offset in pixels from the top left of the cell. This is + /// clamped to the size of a cell. + cell_offset_x: u32, + cell_offset_y: u32, + + /// The source rectangle of the placement. + source_x: u32, + source_y: u32, + source_width: u32, + source_height: u32, +}; + +/// The map used for storing images. +pub const ImageMap = std.AutoHashMapUnmanaged(u32, Image); + +/// The state for a single image that is to be rendered. The image can be +/// pending upload or ready to use with a texture. +pub const Image = union(enum) { + /// The image is pending upload to the GPU. The different keys are + /// different formats since some formats aren't accepted by the GPU + /// and require conversion. + /// + /// This data is owned by this union so it must be freed once the + /// image is uploaded. + pending_rgb: Pending, + pending_rgba: Pending, + + /// The image is uploaded and ready to be used. + ready: gl.Texture, + + /// The image is uploaded but is scheduled to be unloaded. + unload_pending: []u8, + unload_ready: gl.Texture, + + /// Pending image data that needs to be uploaded to the GPU. + pub const Pending = struct { + height: u32, + width: u32, + + /// Data is always expected to be (width * height * depth). Depth + /// is based on the union key. + data: [*]u8, + + pub fn dataSlice(self: Pending, d: u32) []u8 { + return self.data[0..self.len(d)]; + } + + pub fn len(self: Pending, d: u32) u32 { + return self.width * self.height * d; + } + }; + + pub fn deinit(self: Image, alloc: Allocator) void { + switch (self) { + .pending_rgb => |p| alloc.free(p.dataSlice(3)), + .pending_rgba => |p| alloc.free(p.dataSlice(4)), + .unload_pending => |data| alloc.free(data), + + .ready, + .unload_ready, + => |tex| tex.destroy(), + } + } + + /// Mark this image for unload whatever state it is in. + pub fn markForUnload(self: *Image) void { + self.* = switch (self.*) { + .unload_pending, + .unload_ready, + => return, + + .ready => |obj| .{ .unload_ready = obj }, + .pending_rgb => |p| .{ .unload_pending = p.dataSlice(3) }, + .pending_rgba => |p| .{ .unload_pending = p.dataSlice(4) }, + }; + } + + /// Returns true if this image is pending upload. + pub fn isPending(self: Image) bool { + return self.pending() != null; + } + + /// Returns true if this image is pending an unload. + pub fn isUnloading(self: Image) bool { + return switch (self) { + .unload_pending, + .unload_ready, + => true, + + .ready, + .pending_rgb, + .pending_rgba, + => false, + }; + } + + /// Converts the image data to a format that can be uploaded to the GPU. + /// If the data is already in a format that can be uploaded, this is a + /// no-op. + pub fn convert(self: *Image, alloc: Allocator) !void { + switch (self.*) { + .ready, + .unload_pending, + .unload_ready, + => unreachable, // invalid + + .pending_rgba => {}, // ready + + // RGB needs to be converted to RGBA because Metal textures + // don't support RGB. + .pending_rgb => |*p| { + // Note: this is the slowest possible way to do this... + const data = p.dataSlice(3); + const pixels = data.len / 3; + var rgba = try alloc.alloc(u8, pixels * 4); + errdefer alloc.free(rgba); + var i: usize = 0; + while (i < pixels) : (i += 1) { + const data_i = i * 3; + const rgba_i = i * 4; + rgba[rgba_i] = data[data_i]; + rgba[rgba_i + 1] = data[data_i + 1]; + rgba[rgba_i + 2] = data[data_i + 2]; + rgba[rgba_i + 3] = 255; + } + + alloc.free(data); + p.data = rgba.ptr; + self.* = .{ .pending_rgba = p.* }; + }, + } + } + + /// Upload the pending image to the GPU and change the state of this + /// image to ready. + pub fn upload( + self: *Image, + alloc: Allocator, + ) !void { + // Convert our data if we have to + try self.convert(alloc); + + // Get our pending info + const p = self.pending().?; + + // Get our format + const formats: struct { + internal: gl.Texture.InternalFormat, + format: gl.Texture.Format, + } = switch (self.*) { + .pending_rgb => .{ .internal = .rgb, .format = .rgb }, + .pending_rgba => .{ .internal = .rgba, .format = .rgba }, + else => unreachable, + }; + + // Create our texture + const tex = try gl.Texture.create(); + errdefer tex.destroy(); + + const texbind = try tex.bind(.@"2D"); + try texbind.parameter(.WrapS, gl.c.GL_CLAMP_TO_EDGE); + try texbind.parameter(.WrapT, gl.c.GL_CLAMP_TO_EDGE); + try texbind.parameter(.MinFilter, gl.c.GL_LINEAR); + try texbind.parameter(.MagFilter, gl.c.GL_LINEAR); + try texbind.image2D( + 0, + formats.internal, + @intCast(p.width), + @intCast(p.height), + 0, + formats.format, + .UnsignedByte, + p.data, + ); + + // Uploaded. We can now clear our data and change our state. + self.deinit(alloc); + self.* = .{ .ready = tex }; + } + + /// Our pixel depth + fn depth(self: Image) u32 { + return switch (self) { + .pending_rgb => 3, + .pending_rgba => 4, + else => unreachable, + }; + } + + /// Returns true if this image is in a pending state and requires upload. + fn pending(self: Image) ?Pending { + return switch (self) { + .pending_rgb, + .pending_rgba, + => |p| p, + + else => null, + }; + } +};