From 3b55452f57c364a0fd34ddbdd8a0a210cd134f38 Mon Sep 17 00:00:00 2001 From: yunusey Date: Tue, 31 Dec 2024 16:50:47 -0500 Subject: [PATCH] Add background image support for OpenGL --- src/config/Config.zig | 19 ++ src/renderer/OpenGL.zig | 172 +++++++++++++++++- .../opengl/BackgroundImageProgram.zig | 121 ++++++++++++ src/renderer/shaders/bgimage.f.glsl | 13 ++ src/renderer/shaders/bgimage.v.glsl | 40 ++++ src/terminal/kitty/graphics.zig | 1 + 6 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 src/renderer/opengl/BackgroundImageProgram.zig create mode 100644 src/renderer/shaders/bgimage.f.glsl create mode 100644 src/renderer/shaders/bgimage.v.glsl diff --git a/src/config/Config.zig b/src/config/Config.zig index 6f1e89d41..80c67e03e 100644 --- a/src/config/Config.zig +++ b/src/config/Config.zig @@ -20,6 +20,7 @@ const global_state = &@import("../global.zig").state; const fontpkg = @import("../font/main.zig"); const inputpkg = @import("../input.zig"); const terminal = @import("../terminal/main.zig"); +const BackgroundImageProgram = @import("../renderer/opengl/BackgroundImageProgram.zig"); const internal_os = @import("../os/main.zig"); const cli = @import("../cli.zig"); @@ -459,6 +460,24 @@ background: Color = .{ .r = 0x28, .g = 0x2C, .b = 0x34 }, /// Specified as either hex (`#RRGGBB` or `RRGGBB`) or a named X11 color. foreground: Color = .{ .r = 0xFF, .g = 0xFF, .b = 0xFF }, +/// Background image for the window. +@"background-image": RepeatablePath = .{}, + +/// Background image opactity +@"background-image-opacity": f32 = 0.0, + +/// Background image mode to use. +/// +/// `aspect` keeps the aspect-ratio of the background image and `scaled` scales +/// the image to fit the window. `aspect` is the default mode. +/// +/// Valid values are: +/// +/// * `aspect` +/// * `scaled` +/// +@"background-image-mode": BackgroundImageProgram.BackgroundMode = .aspect, + /// The foreground and background color for selection. If this is not set, then /// the selection color is just the inverted window background and foreground /// (note: not to be confused with the cell bg/fg). diff --git a/src/renderer/OpenGL.zig b/src/renderer/OpenGL.zig index d0222a390..490191afd 100644 --- a/src/renderer/OpenGL.zig +++ b/src/renderer/OpenGL.zig @@ -24,6 +24,7 @@ const math = @import("../math.zig"); const Surface = @import("../Surface.zig"); const CellProgram = @import("opengl/CellProgram.zig"); +const BackgroundImageProgram = @import("opengl/BackgroundImageProgram.zig"); const ImageProgram = @import("opengl/ImageProgram.zig"); const gl_image = @import("opengl/image.zig"); const custom = @import("opengl/custom.zig"); @@ -135,6 +136,19 @@ draw_mutex: DrawMutex = drawMutexZero, /// terminal is in reversed mode. draw_background: terminal.color.RGB, +/// The background image(s) to draw. Currentlly, we always draw the last image. +background_image: configpkg.RepeatablePath, + +/// The opacity of the background image. Not to be confused with background-opacity +background_image_opacity: f32, + +/// The background image mode to use. +background_image_mode: BackgroundImageProgram.BackgroundMode, + +/// The current background image to draw. If it is null, then we will not +/// draw any background image. +current_background_image: ?Image = null, + /// Whether we're doing padding extension for vertical sides. padding_extend_top: bool = true, padding_extend_bottom: bool = true, @@ -183,7 +197,7 @@ const SetScreenSize = struct { ); // Update the projection uniform within our shader - inline for (.{ "cell_program", "image_program" }) |name| { + inline for (.{ "cell_program", "image_program", "bgimage_program" }) |name| { const program = @field(gl_state, name); const bind = try program.program.use(); defer bind.unbind(); @@ -281,6 +295,9 @@ pub const DerivedConfig = struct { cursor_opacity: f64, background: terminal.color.RGB, background_opacity: f64, + background_image: configpkg.RepeatablePath, + background_image_opacity: f32, + background_image_mode: BackgroundImageProgram.BackgroundMode, foreground: terminal.color.RGB, selection_background: ?terminal.color.RGB, selection_foreground: ?terminal.color.RGB, @@ -302,6 +319,9 @@ pub const DerivedConfig = struct { // Copy our shaders const custom_shaders = try config.@"custom-shader".clone(alloc); + // Copy our background image + const background_image = try config.@"background-image".clone(alloc); + // Copy our font features const font_features = try config.@"font-feature".clone(alloc); @@ -342,6 +362,11 @@ pub const DerivedConfig = struct { .background = config.background.toTerminalRGB(), .foreground = config.foreground.toTerminalRGB(), + + .background_image = background_image, + .background_image_opacity = config.@"background-image-opacity", + .background_image_mode = config.@"background-image-mode", + .invert_selection_fg_bg = config.@"selection-invert-fg-bg", .bold_is_bright = config.@"bold-is-bright", .min_contrast = @floatCast(config.@"minimum-contrast"), @@ -406,6 +431,9 @@ pub fn init(alloc: Allocator, options: renderer.Options) !OpenGL { .default_background_color = options.config.background, .cursor_color = null, .default_cursor_color = options.config.cursor_color, + .background_image = options.config.background_image, + .background_image_opacity = options.config.background_image_opacity, + .background_image_mode = options.config.background_image_mode, .cursor_invert = options.config.cursor_invert, .surface_mailbox = options.surface_mailbox, .deferred_font_size = .{ .metrics = grid.metrics }, @@ -795,6 +823,14 @@ pub fn updateFrame( try self.prepKittyGraphics(state.terminal); } + if (self.current_background_image == null and + self.background_image.value.items.len > 0) + { + if (single_threaded_draw) self.draw_mutex.lock(); + defer if (single_threaded_draw) self.draw_mutex.unlock(); + try self.prepBackgroundImage(); + } + // If we have any terminal dirty flags set then we need to rebuild // the entire screen. This can be optimized in the future. const full_rebuild: bool = rebuild: { @@ -1160,6 +1196,57 @@ fn prepKittyImage( gop.value_ptr.transmit_time = image.transmit_time; } +/// Prepares the current background image for upload +pub fn prepBackgroundImage(self: *OpenGL) !void { + // If the user doesn't have a background image, do nothing... + const last_image = self.background_image.value.getLastOrNull() orelse return; + + // Get the last background image + const path = switch (last_image) { + .optional, .required => |path| path, + }; + const command = terminal.kitty.graphics.Command{ + .control = .{ + .transmit = .{ + .format = .png, + .medium = .file, + .width = 0, + .height = 0, + .compression = .none, + .image_id = 0, + }, + }, + .data = try self.alloc.dupe(u8, path), + }; + defer command.deinit(self.alloc); + + // Load the iamge + var loading = try terminal.kitty.graphics.LoadingImage.init(self.alloc, &command); + defer loading.deinit(self.alloc); + + // Complete the image to get the final data + var image = try loading.complete(self.alloc); + defer image.deinit(self.alloc); + + // Copy the data into the pending state. + const data = try self.alloc.dupe(u8, image.data); + errdefer self.alloc.free(data); + + const pending: Image.Pending = .{ + .width = image.width, + .height = image.height, + .data = data.ptr, + }; + + self.current_background_image = switch (image.format) { + .gray => .{ .pending_gray = pending }, + .gray_alpha => .{ .pending_gray_alpha = pending }, + .rgb => .{ .pending_rgb = pending }, + .rgba => .{ .pending_rgba = pending }, + .png => unreachable, // should be decoded by now + }; +} + /// rebuildCells rebuilds all the GPU cells from our CPU state. This is a /// slow operation but ensures that the GPU state exactly matches the CPU state. /// In steady-state operation, we use some GPU tricks to send down stale data @@ -2160,6 +2247,14 @@ pub fn changeConfig(self: *OpenGL, config: *DerivedConfig) !void { self.default_cursor_color = if (!config.cursor_invert) config.cursor_color else null; self.cursor_invert = config.cursor_invert; + // Reset current background image + self.background_image = config.background_image; + self.background_image_opacity = config.background_image_opacity; + self.background_image_mode = config.background_image_mode; + if (self.current_background_image) |*img| { + img.markForUnload(); + } + // Update our uniforms self.deferred_config = .{}; @@ -2298,6 +2393,31 @@ pub fn drawFrame(self: *OpenGL, surface: *apprt.Surface) !void { } } + // Check if we need to update our current background image + if (self.current_background_image != null) { + switch (self.current_background_image.?) { + .ready => {}, + + .pending_gray, + .pending_gray_alpha, + .pending_rgb, + .pending_rgba, + .replace_gray, + .replace_gray_alpha, + .replace_rgb, + .replace_rgba, + => try self.current_background_image.?.upload(self.alloc), + + .unload_pending, + .unload_replace, + .unload_ready, + => { + self.current_background_image.?.deinit(self.alloc); + self.current_background_image = null; + }, + } + } + // In the "OpenGL Programming Guide for Mac" it explains that: "When you // use an NSOpenGLView object with OpenGL calls that are issued from a // thread other than the main one, you must set up mutex locking." @@ -2422,6 +2542,9 @@ fn drawCellProgram( ); } + // Draw our background image if defined + try self.drawBackgroundImage(gl_state); + // Draw background images first try self.drawImages( gl_state, @@ -2447,6 +2570,46 @@ fn drawCellProgram( ); } +fn drawBackgroundImage( + self: *OpenGL, + gl_state: *const GLState, +) !void { + // If we don't have a background image, just return + if (self.current_background_image == null) { + return; + } + // Bind our background image program + const bind = try gl_state.bgimage_program.bind(); + defer bind.unbind(); + + // Get the texture + const texture = switch (self.current_background_image.?) { + .ready => |t| t, + else => { + return; + }, + }; + + // Bind the texture + try gl.Texture.active(gl.c.GL_TEXTURE0); + var texbind = try texture.bind(.@"2D"); + defer texbind.unbind(); + + try bind.vbo.setData(BackgroundImageProgram.Input{ + .terminal_width = self.size.terminal().width, + .terminal_height = self.size.terminal().height, + .mode = self.background_image_mode, + }, .static_draw); + try gl_state.bgimage_program.program.setUniform("opacity", self.config.background_image_opacity); + + try gl.drawElementsInstanced( + gl.c.GL_TRIANGLES, + 6, + gl.c.GL_UNSIGNED_BYTE, + 1, + ); +} + /// Runs the image program to draw images. fn drawImages( self: *OpenGL, @@ -2572,6 +2735,7 @@ fn drawCells( /// easy to create/destroy these as a set in situations i.e. where the /// OpenGL context is replaced. const GLState = struct { + bgimage_program: BackgroundImageProgram, cell_program: CellProgram, image_program: ImageProgram, texture: gl.Texture, @@ -2657,6 +2821,10 @@ const GLState = struct { ); } + // Build our background image renderer + const bgimage_program = try BackgroundImageProgram.init(); + errdefer bgimage_program.deinit(); + // Build our cell renderer const cell_program = try CellProgram.init(); errdefer cell_program.deinit(); @@ -2666,6 +2834,7 @@ const GLState = struct { errdefer image_program.deinit(); return .{ + .bgimage_program = bgimage_program, .cell_program = cell_program, .image_program = image_program, .texture = tex, @@ -2678,6 +2847,7 @@ const GLState = struct { if (self.custom) |v| v.deinit(alloc); self.texture.destroy(); self.texture_color.destroy(); + self.bgimage_program.deinit(); self.image_program.deinit(); self.cell_program.deinit(); } diff --git a/src/renderer/opengl/BackgroundImageProgram.zig b/src/renderer/opengl/BackgroundImageProgram.zig new file mode 100644 index 000000000..ec3bf9feb --- /dev/null +++ b/src/renderer/opengl/BackgroundImageProgram.zig @@ -0,0 +1,121 @@ +/// The OpenGL program for rendering terminal cells. +const BackgroundImageProgram = @This(); + +const std = @import("std"); +const gl = @import("opengl"); + +pub const Input = extern struct { + /// vec2 terminal_size + terminal_width: u32 = 0, + terminal_height: u32 = 0, + + /// uint mode + mode: BackgroundMode = .aspect, +}; + +pub const BackgroundMode = enum(u8) { + aspect = 0, + scaled = 1, + _, +}; + +program: gl.Program, +vao: gl.VertexArray, +ebo: gl.Buffer, +vbo: gl.Buffer, + +pub fn init() !BackgroundImageProgram { + // Load and compile our shaders. + const program = try gl.Program.createVF( + @embedFile("../shaders/bgimage.v.glsl"), + @embedFile("../shaders/bgimage.f.glsl"), + ); + errdefer program.destroy(); + + // Set our program uniforms + const pbind = try program.use(); + defer pbind.unbind(); + + // Set all of our texture indexes + try program.setUniform("image", 0); + + // Setup our VAO + const vao = try gl.VertexArray.create(); + errdefer vao.destroy(); + const vaobind = try vao.bind(); + defer vaobind.unbind(); + + // Element buffer (EBO) + const ebo = try gl.Buffer.create(); + errdefer ebo.destroy(); + var ebobind = try ebo.bind(.element_array); + defer ebobind.unbind(); + try ebobind.setData([6]u8{ + 0, 1, 3, // Top-left triangle + 1, 2, 3, // Bottom-right triangle + }, .static_draw); + + // Vertex buffer (VBO) + const vbo = try gl.Buffer.create(); + errdefer vbo.destroy(); + var vbobind = try vbo.bind(.array); + defer vbobind.unbind(); + var offset: usize = 0; + try vbobind.attributeAdvanced(0, 2, gl.c.GL_UNSIGNED_INT, false, @sizeOf(Input), offset); + offset += 2 * @sizeOf(u32); + try vbobind.attributeAdvanced(1, 2, gl.c.GL_UNSIGNED_BYTE, false, @sizeOf(Input), offset); + offset += 1 * @sizeOf(u8); + try vbobind.enableAttribArray(0); + try vbobind.enableAttribArray(1); + try vbobind.attributeDivisor(0, 1); + try vbobind.attributeDivisor(1, 1); + + return .{ + .program = program, + .vao = vao, + .ebo = ebo, + .vbo = vbo, + }; +} + +pub fn bind(self: BackgroundImageProgram) !Binding { + const program = try self.program.use(); + errdefer program.unbind(); + + const vao = try self.vao.bind(); + errdefer vao.unbind(); + + const ebo = try self.ebo.bind(.element_array); + errdefer ebo.unbind(); + + const vbo = try self.vbo.bind(.array); + errdefer vbo.unbind(); + + return .{ + .program = program, + .vao = vao, + .ebo = ebo, + .vbo = vbo, + }; +} + +pub fn deinit(self: BackgroundImageProgram) void { + self.ebo.destroy(); + self.vao.destroy(); + self.vbo.destroy(); + self.program.destroy(); +} + +pub const Binding = struct { + program: gl.Program.Binding, + vao: gl.VertexArray.Binding, + ebo: gl.Buffer.Binding, + vbo: gl.Buffer.Binding, + + pub fn unbind(self: Binding) void { + self.ebo.unbind(); + self.vao.unbind(); + self.vbo.unbind(); + self.program.unbind(); + } +}; diff --git a/src/renderer/shaders/bgimage.f.glsl b/src/renderer/shaders/bgimage.f.glsl new file mode 100644 index 000000000..816882c7c --- /dev/null +++ b/src/renderer/shaders/bgimage.f.glsl @@ -0,0 +1,13 @@ +#version 330 core + +in vec2 tex_coord; + +layout(location = 0) out vec4 out_FragColor; + +uniform sampler2D image; +uniform float opacity; + +void main() { + vec4 color = texture(image, tex_coord); + out_FragColor = vec4(color.rgb * color.a * opacity, color.a * opacity); +} diff --git a/src/renderer/shaders/bgimage.v.glsl b/src/renderer/shaders/bgimage.v.glsl new file mode 100644 index 000000000..dc4bdeec7 --- /dev/null +++ b/src/renderer/shaders/bgimage.v.glsl @@ -0,0 +1,40 @@ +#version 330 core + +const uint MODE_ASPECT = 0u; +const uint MODE_SCALED = 1u; + +layout (location = 0) in vec2 terminal_size; +layout (location = 1) in uint mode; + +out vec2 tex_coord; +uniform sampler2D image; +uniform mat4 projection; + +void main() { + vec2 position; + position.x = (gl_VertexID == 0 || gl_VertexID == 1) ? 1. : 0.; + position.y = (gl_VertexID == 0 || gl_VertexID == 3) ? 0. : 1.; + + vec2 image_size = textureSize(image, 0); + vec2 scale = vec2(1.0, 1.0); + switch (mode) { + case MODE_ASPECT: + vec2 aspect_ratio = vec2( + terminal_size.x / terminal_size.y, + image_size.x / image_size.y + ); + if (aspect_ratio.x > aspect_ratio.y) { + scale.x = aspect_ratio.y / aspect_ratio.x; + } + else { + scale.y = aspect_ratio.x / aspect_ratio.y; + } + case MODE_SCALED: + break; + } + + vec2 image_pos = terminal_size * position * scale; + vec2 offset = (terminal_size * (1.0 - scale)) / 2.0; + gl_Position = projection * vec4(image_pos.xy + offset, 0.0, 1.0); + tex_coord = position; +} diff --git a/src/terminal/kitty/graphics.zig b/src/terminal/kitty/graphics.zig index c710f81a1..ee2ef7804 100644 --- a/src/terminal/kitty/graphics.zig +++ b/src/terminal/kitty/graphics.zig @@ -24,6 +24,7 @@ const storage = @import("graphics_storage.zig"); pub const unicode = @import("graphics_unicode.zig"); pub const Command = command.Command; pub const CommandParser = command.Parser; +pub const LoadingImage = image.LoadingImage; pub const Image = image.Image; pub const ImageStorage = storage.ImageStorage; pub const RenderPlacement = render.Placement;