Merge pull request #12 from mitchellh/ligs

Ligatures v1 and VS15/VS16 Emoji Support

TODO.md

@@ -12,6 +12,7 @@ Performance:
   screen data structure.
 * Screen cell structure should be rethought to use some data oriented design,
   also bring it closer to GPU cells, perhaps.
+* Cache text shaping results and only invalidate if the line becomes dirty.
 
 Correctness:
 
@@ -36,7 +37,6 @@ Improvements:
 Major Features:
 
 * Strikethrough
-* Ligatures
 * Bell
 * Mac:
   - Switch to raw Cocoa and Metal instead of glfw and libuv (major!)

shaders/cell.v.glsl

@@ -11,7 +11,6 @@ const uint MODE_CURSOR_RECT = 3u;
 const uint MODE_CURSOR_RECT_HOLLOW = 4u;
 const uint MODE_CURSOR_BAR = 5u;
 const uint MODE_UNDERLINE = 6u;
-const uint MODE_WIDE_MASK = 128u; // 0b1000_0000
 
 // The grid coordinates (x, y) where x < columns and y < rows
 layout (location = 0) in vec2 grid_coord;
@@ -37,6 +36,9 @@ layout (location = 5) in vec4 bg_color_in;
 // the entire terminal grid in a single GPU pass.
 layout (location = 6) in uint mode_in;
 
+// The width in cells of this item.
+layout (location = 7) in uint grid_width;
+
 // The background or foreground color for the fragment, depending on
 // whether this is a background or foreground pass.
 flat out vec4 color;
@@ -78,12 +80,9 @@ uniform float glyph_baseline;
  */
 
 void main() {
-    // Remove any masks from our mode
-    uint mode_unmasked = mode_in & ~MODE_WIDE_MASK;
-
     // We always forward our mode unmasked because the fragment
     // shader doesn't use any of the masks.
-    mode = mode_unmasked;
+    mode = mode_in;
 
     // Top-left cell coordinates converted to world space
     // Example: (1,0) with a 30 wide cell is converted to (30,0)
@@ -110,9 +109,7 @@ void main() {
 
     // Scaled for wide chars
     vec2 cell_size_scaled = cell_size;
-    if ((mode_in & MODE_WIDE_MASK) == MODE_WIDE_MASK) {
+    cell_size_scaled.x = cell_size_scaled.x * grid_width;
-        cell_size_scaled.x = cell_size_scaled.x * 2;
-    }
 
     switch (mode) {
     case MODE_BG:
@@ -133,10 +130,10 @@ void main() {
         // The "+ 3" here is to give some wiggle room for fonts that are
         // BARELY over it.
         vec2 glyph_size_downsampled = glyph_size;
-        if (glyph_size.x > (cell_size.x + 3)) {
+        if (glyph_size_downsampled.y > cell_size_scaled.y + 2) {
-            glyph_size_downsampled.x = cell_size_scaled.x;
+            glyph_size_downsampled.y = cell_size_scaled.y;
-            glyph_size_downsampled.y = glyph_size.y * (glyph_size_downsampled.x / glyph_size.x);
+            glyph_size_downsampled.x = glyph_size.x * (glyph_size_downsampled.y / glyph_size.y);
-            glyph_offset_calc.y = glyph_offset.y * (glyph_size_downsampled.x / glyph_size.x);
+            glyph_offset_calc.y = glyph_offset.y * (glyph_size_downsampled.y / glyph_size.y);
         }
 
         // The glyph_offset.y is the y bearing, a y value that when added

src/Atlas.zig

@@ -48,10 +48,18 @@ modified: bool = false,
 /// updated in-place.
 resized: bool = false,
 
-pub const Format = enum(u3) {
+pub const Format = enum {
-    greyscale = 1,
+    greyscale,
-    rgb = 3,
+    rgb,
-    rgba = 4,
+    rgba,
+
+    pub fn depth(self: Format) u8 {
+        return switch (self) {
+            .greyscale => 1,
+            .rgb => 3,
+            .rgba => 4,
+        };
+    }
 };
 
 const Node = struct {
@@ -76,7 +84,7 @@ pub const Region = struct {
 
 pub fn init(alloc: Allocator, size: u32, format: Format) !Atlas {
     var result = Atlas{
-        .data = try alloc.alloc(u8, size * size * @enumToInt(format)),
+        .data = try alloc.alloc(u8, size * size * format.depth()),
         .size = size,
         .nodes = .{},
         .format = format,
@@ -220,7 +228,7 @@ pub fn set(self: *Atlas, reg: Region, data: []const u8) void {
     assert(reg.y < (self.size - 1));
     assert((reg.y + reg.height) <= (self.size - 1));
 
-    const depth = @enumToInt(self.format);
+    const depth = self.format.depth();
     var i: u32 = 0;
     while (i < reg.height) : (i += 1) {
         const tex_offset = (((reg.y + i) * self.size) + reg.x) * depth;
@@ -245,7 +253,7 @@ pub fn grow(self: *Atlas, alloc: Allocator, size_new: u32) Allocator.Error!void
     const size_old = self.size;
 
     // Allocate our new data
-    self.data = try alloc.alloc(u8, size_new * size_new * @enumToInt(self.format));
+    self.data = try alloc.alloc(u8, size_new * size_new * self.format.depth());
     defer alloc.free(data_old);
     errdefer {
         alloc.free(self.data);
@@ -270,7 +278,7 @@ pub fn grow(self: *Atlas, alloc: Allocator, size_new: u32) Allocator.Error!void
         .y = 1, // skip the first border row
         .width = size_old,
         .height = size_old - 2, // skip the last border row
-    }, data_old[size_old * @enumToInt(self.format) ..]);
+    }, data_old[size_old * self.format.depth() ..]);
 
     // We are both modified and resized
     self.modified = true;
@@ -380,7 +388,7 @@ test "writing RGB data" {
     });
 
     // 33 because of the 1px border and so on
-    const depth = @intCast(usize, @enumToInt(atlas.format));
+    const depth = @intCast(usize, atlas.format.depth());
     try testing.expectEqual(@as(u8, 1), atlas.data[33 * depth]);
     try testing.expectEqual(@as(u8, 2), atlas.data[33 * depth + 1]);
     try testing.expectEqual(@as(u8, 3), atlas.data[33 * depth + 2]);
@@ -410,7 +418,7 @@ test "grow RGB" {
 
     // Our top left skips the first row (size * depth) and the first
     // column (depth) for the 1px border.
-    const depth = @intCast(usize, @enumToInt(atlas.format));
+    const depth = @intCast(usize, atlas.format.depth());
     var tl = (atlas.size * depth) + depth;
     try testing.expectEqual(@as(u8, 10), atlas.data[tl]);
     try testing.expectEqual(@as(u8, 11), atlas.data[tl + 1]);

src/Grid.zig

@@ -78,7 +78,9 @@ pub const CursorStyle = enum(u8) {
 };
 
 /// The raw structure that maps directly to the buffer sent to the vertex shader.
-const GPUCell = struct {
+/// This must be "extern" so that the field order is not reordered by the
+/// Zig compiler.
+const GPUCell = extern struct {
     /// vec2 grid_coord
     grid_col: u16,
     grid_row: u16,
@@ -109,6 +111,9 @@ const GPUCell = struct {
 
     /// uint mode
     mode: GPUCellMode,
+
+    /// The width in grid cells that a rendering takes.
+    grid_width: u8,
 };
 
 const GPUCellMode = enum(u8) {
@@ -120,8 +125,6 @@ const GPUCellMode = enum(u8) {
     cursor_bar = 5,
     underline = 6,
 
-    wide_mask = 0b1000_0000,
-
     // Non-exhaustive because masks change it
     _,
 
@@ -163,6 +166,11 @@ pub fn init(
             .regular,
             try font.Face.init(font_lib, face_emoji_ttf, font_size),
         );
+        try group.addFace(
+            alloc,
+            .regular,
+            try font.Face.init(font_lib, face_emoji_text_ttf, font_size),
+        );
 
         break :group group;
     });
@@ -224,6 +232,8 @@ pub fn init(
     try vbobind.attributeAdvanced(5, 4, gl.c.GL_UNSIGNED_BYTE, false, @sizeOf(GPUCell), offset);
     offset += 4 * @sizeOf(u8);
     try vbobind.attributeIAdvanced(6, 1, gl.c.GL_UNSIGNED_BYTE, @sizeOf(GPUCell), offset);
+    offset += 1 * @sizeOf(u8);
+    try vbobind.attributeIAdvanced(7, 1, gl.c.GL_UNSIGNED_BYTE, @sizeOf(GPUCell), offset);
     try vbobind.enableAttribArray(0);
     try vbobind.enableAttribArray(1);
     try vbobind.enableAttribArray(2);
@@ -231,6 +241,7 @@ pub fn init(
     try vbobind.enableAttribArray(4);
     try vbobind.enableAttribArray(5);
     try vbobind.enableAttribArray(6);
+    try vbobind.enableAttribArray(7);
     try vbobind.attributeDivisor(0, 1);
     try vbobind.attributeDivisor(1, 1);
     try vbobind.attributeDivisor(2, 1);
@@ -238,6 +249,7 @@ pub fn init(
     try vbobind.attributeDivisor(4, 1);
     try vbobind.attributeDivisor(5, 1);
     try vbobind.attributeDivisor(6, 1);
+    try vbobind.attributeDivisor(7, 1);
 
     // Build our texture
     const tex = try gl.Texture.create();
@@ -341,17 +353,31 @@ pub fn rebuildCells(self: *Grid, term: *Terminal) !void {
     // We've written no data to the GPU, refresh it all
     self.gl_cells_written = 0;
 
+    // Create a text shaper we'll use for the screen
+    var shape_buf = try self.alloc.alloc(font.Shaper.Cell, term.screen.cols * 2);
+    defer self.alloc.free(shape_buf);
+    var shaper = try font.Shaper.init(&self.font_group, shape_buf);
+    defer shaper.deinit();
+
     // Build each cell
     var rowIter = term.screen.rowIterator(.viewport);
     var y: usize = 0;
     while (rowIter.next()) |row| {
         defer y += 1;
 
-        var cellIter = row.cellIterator();
+        // Split our row into runs and shape each one.
-        var x: usize = 0;
+        var iter = shaper.runIterator(row);
-        while (cellIter.next()) |cell| {
+        while (try iter.next(self.alloc)) |run| {
-            defer x += 1;
+            for (try shaper.shape(run)) |shaper_cell| {
-            assert(try self.updateCell(term, cell, x, y));
+                assert(try self.updateCell(
+                    term,
+                    row.getCell(shaper_cell.x),
+                    shaper_cell,
+                    run,
+                    shaper_cell.x,
+                    y,
+                ));
+            }
         }
     }
 
@@ -392,12 +418,12 @@ fn addCursor(self: *Grid, term: *Terminal) void {
             GPUCellMode,
             @enumToInt(self.cursor_style),
         );
-        if (cell.attrs.wide) mode = mode.mask(.wide_mask);
 
         self.cells.appendAssumeCapacity(.{
             .mode = mode,
             .grid_col = @intCast(u16, term.screen.cursor.x),
             .grid_row = @intCast(u16, term.screen.cursor.y),
+            .grid_width = if (cell.attrs.wide) 2 else 1,
             .fg_r = 0,
             .fg_g = 0,
             .fg_b = 0,
@@ -417,6 +443,8 @@ pub fn updateCell(
     self: *Grid,
     term: *Terminal,
     cell: terminal.Screen.Cell,
+    shaper_cell: font.Shaper.Cell,
+    shaper_run: font.Shaper.TextRun,
     x: usize,
     y: usize,
 ) !bool {
@@ -471,9 +499,6 @@ pub fn updateCell(
         break :colors res;
     };
 
-    // If we are a trailing spacer, we never render anything.
-    if (cell.attrs.wide_spacer_tail) return true;
-
     // Calculate the amount of space we need in the cells list.
     const needed = needed: {
         var i: usize = 0;
@@ -490,12 +515,12 @@ pub fn updateCell(
     // If the cell has a background, we always draw it.
     if (colors.bg) |rgb| {
         var mode: GPUCellMode = .bg;
-        if (cell.attrs.wide) mode = mode.mask(.wide_mask);
 
         self.cells.appendAssumeCapacity(.{
             .mode = mode,
             .grid_col = @intCast(u16, x),
             .grid_row = @intCast(u16, y),
+            .grid_width = shaper_cell.width,
             .glyph_x = 0,
             .glyph_y = 0,
             .glyph_width = 0,
@@ -515,46 +540,23 @@ pub fn updateCell(
 
     // If the cell is empty then we draw nothing in the box.
     if (!cell.empty()) {
-        // Determine our glyph styling
-        const style: font.Style = if (cell.attrs.bold)
-            .bold
-        else
-            .regular;
-
-        var mode: GPUCellMode = .fg;
-
-        // Get the glyph that we're going to use. We first try what the cell
-        // wants, then the Unicode replacement char, then finally a space.
-        const FontInfo = struct { index: font.Group.FontIndex, ch: u32 };
-        const font_info: FontInfo = font_info: {
-            var chars = [_]u32{ @intCast(u32, cell.char), 0xFFFD, ' ' };
-            for (chars) |char| {
-                if (try self.font_group.indexForCodepoint(self.alloc, style, char)) |idx| {
-                    break :font_info FontInfo{
-                        .index = idx,
-                        .ch = char,
-                    };
-                }
-            }
-
-            @panic("all fonts require at least space");
-        };
-
         // Render
-        const face = self.font_group.group.faceFromIndex(font_info.index);
+        const face = self.font_group.group.faceFromIndex(shaper_run.font_index);
-        const glyph_index = face.glyphIndex(font_info.ch).?;
+        const glyph = try self.font_group.renderGlyph(
-        const glyph = try self.font_group.renderGlyph(self.alloc, font_info.index, glyph_index);
+            self.alloc,
+            shaper_run.font_index,
+            shaper_cell.glyph_index,
+        );
 
         // If we're rendering a color font, we use the color atlas
+        var mode: GPUCellMode = .fg;
         if (face.hasColor()) mode = .fg_color;
 
-        // If the cell is wide, we need to note that in the mode
-        if (cell.attrs.wide) mode = mode.mask(.wide_mask);
-
         self.cells.appendAssumeCapacity(.{
             .mode = mode,
             .grid_col = @intCast(u16, x),
             .grid_row = @intCast(u16, y),
+            .grid_width = shaper_cell.width,
             .glyph_x = glyph.atlas_x,
             .glyph_y = glyph.atlas_y,
             .glyph_width = glyph.width,
@@ -573,13 +575,11 @@ pub fn updateCell(
     }
 
     if (cell.attrs.underline) {
-        var mode: GPUCellMode = .underline;
-        if (cell.attrs.wide) mode = mode.mask(.wide_mask);
-
         self.cells.appendAssumeCapacity(.{
-            .mode = mode,
+            .mode = .underline,
             .grid_col = @intCast(u16, x),
             .grid_row = @intCast(u16, y),
+            .grid_width = shaper_cell.width,
             .glyph_x = 0,
             .glyph_y = 0,
             .glyph_width = 0,
@@ -835,3 +835,4 @@ test "GridSize update rounding" {
 const face_ttf = @embedFile("font/res/FiraCode-Regular.ttf");
 const face_bold_ttf = @embedFile("font/res/FiraCode-Bold.ttf");
 const face_emoji_ttf = @embedFile("font/res/NotoColorEmoji.ttf");
+const face_emoji_text_ttf = @embedFile("font/res/NotoEmoji-Regular.ttf");

src/font/Face.zig

@@ -15,6 +15,7 @@ const Allocator = std.mem.Allocator;
 const Atlas = @import("../Atlas.zig");
 const Glyph = @import("main.zig").Glyph;
 const Library = @import("main.zig").Library;
+const Presentation = @import("main.zig").Presentation;
 
 const log = std.log.scoped(.font_face);
 
@@ -24,6 +25,10 @@ face: freetype.Face,
 /// Harfbuzz font corresponding to this face.
 hb_font: harfbuzz.Font,
 
+/// The presentation for this font. This is a heuristic since fonts don't have
+/// a way to declare this. We just assume a font with color is an emoji font.
+presentation: Presentation,
+
 /// If a DPI can't be calculated, this DPI is used. This is probably
 /// wrong on modern devices so it is highly recommended you get the DPI
 /// using whatever platform method you can.
@@ -55,7 +60,11 @@ pub fn init(lib: Library, source: [:0]const u8, size: DesiredSize) !Face {
     const hb_font = try harfbuzz.freetype.createFont(face.handle);
     errdefer hb_font.destroy();
 
-    return Face{ .face = face, .hb_font = hb_font };
+    return Face{
+        .face = face,
+        .hb_font = hb_font,
+        .presentation = if (face.hasColor()) .emoji else .text,
+    };
 }
 
 pub fn deinit(self: *Face) void {
@@ -120,30 +129,43 @@ pub fn renderGlyph(self: Face, alloc: Allocator, atlas: *Atlas, glyph_index: u32
     });
 
     const glyph = self.face.handle.*.glyph;
-    const bitmap = glyph.*.bitmap;
+    const bitmap_ft = glyph.*.bitmap;
+
+    // This bitmap is blank. I've seen it happen in a font, I don't know why.
+    // If it is empty, we just return a valid glyph struct that does nothing.
+    if (bitmap_ft.rows == 0) return Glyph{
+        .width = 0,
+        .height = 0,
+        .offset_x = 0,
+        .offset_y = 0,
+        .atlas_x = 0,
+        .atlas_y = 0,
+        .advance_x = 0,
+    };
 
     // Ensure we know how to work with the font format. And assure that
-    // or color depth is as expected on the texture atlas.
+    // or color depth is as expected on the texture atlas. If format is null
-    const format: Atlas.Format = switch (bitmap.pixel_mode) {
+    // it means there is no native color format for our Atlas and we must try
+    // conversion.
+    const format: Atlas.Format = switch (bitmap_ft.pixel_mode) {
         freetype.c.FT_PIXEL_MODE_GRAY => .greyscale,
         freetype.c.FT_PIXEL_MODE_BGRA => .rgba,
         else => {
-            log.warn("pixel mode={}", .{bitmap.pixel_mode});
+            log.warn("glyph={} pixel mode={}", .{ glyph_index, bitmap_ft.pixel_mode });
             @panic("unsupported pixel mode");
         },
     };
     assert(atlas.format == format);
 
-    const src_w = bitmap.width;
+    const bitmap = bitmap_ft;
-    const src_h = bitmap.rows;
+    const tgt_w = bitmap.width;
-    const tgt_w = src_w;
+    const tgt_h = bitmap.rows;
-    const tgt_h = src_h;
 
     const region = try atlas.reserve(alloc, tgt_w, tgt_h);
 
     // If we have data, copy it into the atlas
     if (region.width > 0 and region.height > 0) {
-        const depth = @enumToInt(format);
+        const depth = atlas.format.depth();
 
         // We can avoid a buffer copy if our atlas width and bitmap
         // width match and the bitmap pitch is just the width (meaning
@@ -156,7 +178,7 @@ pub fn renderGlyph(self: Face, alloc: Allocator, atlas: *Atlas, glyph_index: u32
             var dst_ptr = temp;
             var src_ptr = bitmap.buffer;
             var i: usize = 0;
-            while (i < src_h) : (i += 1) {
+            while (i < bitmap.rows) : (i += 1) {
                 std.mem.copy(u8, dst_ptr, src_ptr[0 .. bitmap.width * depth]);
                 dst_ptr = dst_ptr[tgt_w * depth ..];
                 src_ptr += @intCast(usize, bitmap.pitch);
@@ -210,6 +232,8 @@ test {
     var font = try init(lib, testFont, .{ .points = 12 });
     defer font.deinit();
 
+    try testing.expectEqual(Presentation.text, font.presentation);
+
     // Generate all visible ASCII
     var i: u8 = 32;
     while (i < 127) : (i += 1) {
@@ -230,5 +254,24 @@ test "color emoji" {
     var font = try init(lib, testFont, .{ .points = 12 });
     defer font.deinit();
 
+    try testing.expectEqual(Presentation.emoji, font.presentation);
+
     _ = try font.renderGlyph(alloc, &atlas, font.glyphIndex('🥸').?);
 }
+
+test "mono to rgba" {
+    const alloc = testing.allocator;
+    const testFont = @import("test.zig").fontEmoji;
+
+    var lib = try Library.init();
+    defer lib.deinit();
+
+    var atlas = try Atlas.init(alloc, 512, .rgba);
+    defer atlas.deinit(alloc);
+
+    var font = try init(lib, testFont, .{ .points = 12 });
+    defer font.deinit();
+
+    // glyph 3 is mono in Noto
+    _ = try font.renderGlyph(alloc, &atlas, 3);
+}

src/font/Group.zig

@@ -15,6 +15,7 @@ const Face = @import("main.zig").Face;
 const Library = @import("main.zig").Library;
 const Glyph = @import("main.zig").Glyph;
 const Style = @import("main.zig").Style;
+const Presentation = @import("main.zig").Presentation;
 
 const log = std.log.scoped(.font_group);
 
@@ -86,19 +87,34 @@ pub const FontIndex = packed struct {
 /// The font index is valid as long as font faces aren't removed. This
 /// isn't cached; it is expected that downstream users handle caching if
 /// that is important.
-pub fn indexForCodepoint(self: Group, style: Style, cp: u32) ?FontIndex {
+///
+/// Optionally, a presentation format can be specified. This presentation
+/// format will be preferred but if it can't be found in this format,
+/// any text format will be accepted. If presentation is null, any presentation
+/// is allowed. This func will NOT determine the default presentation for
+/// a code point.
+pub fn indexForCodepoint(
+    self: Group,
+    cp: u32,
+    style: Style,
+    p: ?Presentation,
+) ?FontIndex {
     // If we can find the exact value, then return that.
-    if (self.indexForCodepointExact(style, cp)) |value| return value;
+    if (self.indexForCodepointExact(cp, style, p)) |value| return value;
 
     // If this is already regular, we're done falling back.
-    if (style == .regular) return null;
+    if (style == .regular and p == null) return null;
 
     // For non-regular fonts, we fall back to regular.
-    return self.indexForCodepointExact(.regular, cp);
+    return self.indexForCodepointExact(cp, .regular, null);
 }
 
-fn indexForCodepointExact(self: Group, style: Style, cp: u32) ?FontIndex {
+fn indexForCodepointExact(self: Group, cp: u32, style: Style, p: ?Presentation) ?FontIndex {
     for (self.faces.get(style).items) |face, i| {
+        // If the presentation is null, we allow the first presentation we
+        // can find. Otherwise, we check for the specific one requested.
+        if (p != null and face.presentation != p.?) continue;
+
         if (face.glyphIndex(cp) != null) {
             return FontIndex{
                 .style = style,
@@ -143,6 +159,7 @@ test {
     const alloc = testing.allocator;
     const testFont = @import("test.zig").fontRegular;
     const testEmoji = @import("test.zig").fontEmoji;
+    const testEmojiText = @import("test.zig").fontEmojiText;
 
     var atlas_greyscale = try Atlas.init(alloc, 512, .greyscale);
     defer atlas_greyscale.deinit(alloc);
@@ -155,11 +172,12 @@ test {
 
     try group.addFace(alloc, .regular, try Face.init(lib, testFont, .{ .points = 12 }));
     try group.addFace(alloc, .regular, try Face.init(lib, testEmoji, .{ .points = 12 }));
+    try group.addFace(alloc, .regular, try Face.init(lib, testEmojiText, .{ .points = 12 }));
 
     // Should find all visible ASCII
     var i: u32 = 32;
     while (i < 127) : (i += 1) {
-        const idx = group.indexForCodepoint(.regular, i).?;
+        const idx = group.indexForCodepoint(i, .regular, null).?;
         try testing.expectEqual(Style.regular, idx.style);
         try testing.expectEqual(@as(FontIndex.IndexInt, 0), idx.idx);
 
@@ -176,7 +194,19 @@ test {
 
     // Try emoji
     {
-        const idx = group.indexForCodepoint(.regular, '🥸').?;
+        const idx = group.indexForCodepoint('🥸', .regular, null).?;
+        try testing.expectEqual(Style.regular, idx.style);
+        try testing.expectEqual(@as(FontIndex.IndexInt, 1), idx.idx);
+    }
+
+    // Try text emoji
+    {
+        const idx = group.indexForCodepoint(0x270C, .regular, .text).?;
+        try testing.expectEqual(Style.regular, idx.style);
+        try testing.expectEqual(@as(FontIndex.IndexInt, 2), idx.idx);
+    }
+    {
+        const idx = group.indexForCodepoint(0x270C, .regular, .emoji).?;
         try testing.expectEqual(Style.regular, idx.style);
         try testing.expectEqual(@as(FontIndex.IndexInt, 1), idx.idx);
     }

src/font/GroupCache.zig

@@ -12,6 +12,7 @@ const Glyph = @import("main.zig").Glyph;
 const Style = @import("main.zig").Style;
 const Group = @import("main.zig").Group;
 const Metrics = @import("main.zig").Metrics;
+const Presentation = @import("main.zig").Presentation;
 
 const log = std.log.scoped(.font_groupcache);
 
@@ -34,6 +35,7 @@ atlas_color: Atlas,
 const CodepointKey = struct {
     style: Style,
     codepoint: u32,
+    presentation: ?Presentation,
 };
 
 const GlyphKey = struct {
@@ -90,7 +92,7 @@ pub fn metrics(self: *GroupCache, alloc: Allocator) !Metrics {
         var cell_width: f32 = 0;
         var i: u32 = 32;
         while (i <= 126) : (i += 1) {
-            const index = (try self.indexForCodepoint(alloc, .regular, i)).?;
+            const index = (try self.indexForCodepoint(alloc, i, .regular, .text)).?;
             const face = self.group.faceFromIndex(index);
             const glyph_index = face.glyphIndex(i).?;
             const glyph = try self.renderGlyph(alloc, index, glyph_index);
@@ -106,7 +108,7 @@ pub fn metrics(self: *GroupCache, alloc: Allocator) !Metrics {
     // '_' which should live at the bottom of a cell.
     const cell_height: f32 = cell_height: {
         // Get the '_' char for height
-        const index = (try self.indexForCodepoint(alloc, .regular, '_')).?;
+        const index = (try self.indexForCodepoint(alloc, '_', .regular, .text)).?;
         const face = self.group.faceFromIndex(index);
         const glyph_index = face.glyphIndex('_').?;
         const glyph = try self.renderGlyph(alloc, index, glyph_index);
@@ -142,15 +144,21 @@ pub fn metrics(self: *GroupCache, alloc: Allocator) !Metrics {
 }
 
 /// Get the font index for a given codepoint. This is cached.
-pub fn indexForCodepoint(self: *GroupCache, alloc: Allocator, style: Style, cp: u32) !?Group.FontIndex {
+pub fn indexForCodepoint(
-    const key: CodepointKey = .{ .style = style, .codepoint = cp };
+    self: *GroupCache,
+    alloc: Allocator,
+    cp: u32,
+    style: Style,
+    p: ?Presentation,
+) !?Group.FontIndex {
+    const key: CodepointKey = .{ .style = style, .codepoint = cp, .presentation = p };
     const gop = try self.codepoints.getOrPut(alloc, key);
 
     // If it is in the cache, use it.
     if (gop.found_existing) return gop.value_ptr.*;
 
     // Load a value and cache it. This even caches negative matches.
-    const value = self.group.indexForCodepoint(style, cp);
+    const value = self.group.indexForCodepoint(cp, style, p);
     gop.value_ptr.* = value;
     return value;
 }
@@ -219,7 +227,7 @@ test {
     // Visible ASCII. Do it twice to verify cache.
     var i: u32 = 32;
     while (i < 127) : (i += 1) {
-        const idx = (try cache.indexForCodepoint(alloc, .regular, i)).?;
+        const idx = (try cache.indexForCodepoint(alloc, i, .regular, null)).?;
         try testing.expectEqual(Style.regular, idx.style);
         try testing.expectEqual(@as(Group.FontIndex.IndexInt, 0), idx.idx);
 
@@ -240,7 +248,7 @@ test {
 
         i = 32;
         while (i < 127) : (i += 1) {
-            const idx = (try cache.indexForCodepoint(alloc, .regular, i)).?;
+            const idx = (try cache.indexForCodepoint(alloc, i, .regular, null)).?;
             try testing.expectEqual(Style.regular, idx.style);
             try testing.expectEqual(@as(Group.FontIndex.IndexInt, 0), idx.idx);
 

src/font/Shaper.zig

@@ -5,12 +5,14 @@ const std = @import("std");
 const assert = std.debug.assert;
 const Allocator = std.mem.Allocator;
 const harfbuzz = @import("harfbuzz");
+const trace = @import("tracy").trace;
 const Atlas = @import("../Atlas.zig");
 const Face = @import("main.zig").Face;
 const Group = @import("main.zig").Group;
 const GroupCache = @import("main.zig").GroupCache;
 const Library = @import("main.zig").Library;
 const Style = @import("main.zig").Style;
+const Presentation = @import("main.zig").Presentation;
 const terminal = @import("../terminal/main.zig");
 
 const log = std.log.scoped(.font_shaper);
@@ -22,10 +24,16 @@ group: *GroupCache,
 /// calls to prevent allocations.
 hb_buf: harfbuzz.Buffer,
 
-pub fn init(group: *GroupCache) !Shaper {
+/// The shared memory used for shaping results.
+cell_buf: []Cell,
+
+/// The cell_buf argument is the buffer to use for storing shaped results.
+/// This should be at least the number of columns in the terminal.
+pub fn init(group: *GroupCache, cell_buf: []Cell) !Shaper {
     return Shaper{
         .group = group,
         .hb_buf = try harfbuzz.Buffer.create(),
+        .cell_buf = cell_buf,
     };
 }
 
@@ -44,27 +52,96 @@ pub fn runIterator(self: *Shaper, row: terminal.Screen.Row) RunIterator {
 /// text run that was iterated since the text run does share state with the
 /// Shaper struct.
 ///
-/// NOTE: there is no return value here yet because its still WIP
+/// The return value is only valid until the next shape call is called.
-pub fn shape(self: Shaper, run: TextRun) void {
+///
-    const face = self.group.group.faceFromIndex(run.font_index);
+/// If there is not enough space in the cell buffer, an error is returned.
-    harfbuzz.shape(face.hb_font, self.hb_buf, null);
+pub fn shape(self: *Shaper, run: TextRun) ![]Cell {
+    const tracy = trace(@src());
+    defer tracy.end();
 
+    // TODO: we do not want to hardcode these
+    const hb_feats = &[_]harfbuzz.Feature{
+        harfbuzz.Feature.fromString("dlig").?,
+        harfbuzz.Feature.fromString("liga").?,
+    };
+
+    const face = self.group.group.faceFromIndex(run.font_index);
+    harfbuzz.shape(face.hb_font, self.hb_buf, hb_feats);
+
+    // If our buffer is empty, we short-circuit the rest of the work
+    // return nothing.
+    if (self.hb_buf.getLength() == 0) return self.cell_buf[0..0];
     const info = self.hb_buf.getGlyphInfos();
-    const pos = self.hb_buf.getGlyphPositions() orelse return;
+    const pos = self.hb_buf.getGlyphPositions() orelse return error.HarfbuzzFailed;
 
     // This is perhaps not true somewhere, but we currently assume it is true.
     // If it isn't true, I'd like to catch it and learn more.
     assert(info.len == pos.len);
 
-    // log.warn("info={} pos={}", .{ info.len, pos.len });
+    // Convert all our info/pos to cells and set it.
-    // for (info) |v, i| {
+    if (info.len > self.cell_buf.len) return error.OutOfMemory;
-    //     log.warn("info {} = {}", .{ i, v });
+    //log.warn("info={} pos={} run={}", .{ info.len, pos.len, run });
-    // }
+
+    // x is the column that we currently occupy. We start at the offset.
+    var x: u16 = run.offset;
+
+    for (info) |v, i| {
+        // The number of codepoints is used as the cell "width". If
+        // we're the last cell, this is remaining otherwise we use cluster numbers
+        // to detect since we set the cluster number to the column it
+        // originated.
+        const cp_width = if (i == info.len - 1)
+            run.max_cluster - v.cluster
+        else width: {
+            const next_cluster = info[i + 1].cluster;
+            //log.warn("next={}", .{next_cluster});
+            break :width next_cluster - v.cluster;
+        };
+        //log.warn("cluster={} max={}", .{ v.cluster, run.max_cluster });
+
+        self.cell_buf[i] = .{
+            .x = x,
+            .glyph_index = v.codepoint,
+            .width = @intCast(u8, cp_width),
+        };
+
+        // Increase x by the amount of codepoints we replaced so that
+        // we retain the grid.
+        x += @intCast(u16, cp_width);
+
+        //log.warn("i={} info={} pos={} cell={}", .{ i, v, pos[i], self.cell_buf[i] });
     }
 
+    return self.cell_buf[0..info.len];
+}
+
+pub const Cell = struct {
+    /// The column that this cell occupies. Since a set of shaper cells is
+    /// always on the same line, only the X is stored. It is expected the
+    /// caller has access to the original screen cell.
+    x: u16,
+
+    /// The glyph index for this cell. The font index to use alongside
+    /// this cell is available in the text run.
+    glyph_index: u32,
+
+    /// The width that this cell consumes.
+    width: u8,
+};
+
 /// A single text run. A text run is only valid for one Shaper and
 /// until the next run is created.
 pub const TextRun = struct {
+    /// The offset in the row where this run started
+    offset: u16,
+
+    /// The total number of cells produced by this run.
+    cells: u16,
+
+    /// The maximum cluster value used
+    max_cluster: u16,
+
+    /// The font index to use for the glyphs of this run.
     font_index: Group.FontIndex,
 };
 
@@ -74,7 +151,18 @@ pub const RunIterator = struct {
     i: usize = 0,
 
     pub fn next(self: *RunIterator, alloc: Allocator) !?TextRun {
-        if (self.i >= self.row.lenCells()) return null;
+        const tracy = trace(@src());
+        defer tracy.end();
+
+        // Trim the right side of a row that might be empty
+        const max: usize = max: {
+            var j: usize = self.row.lenCells();
+            while (j > 0) : (j -= 1) if (!self.row.getCell(j - 1).empty()) break;
+            break :max j;
+        };
+
+        // We're over at the max
+        if (self.i >= max) return null;
 
         // Track the font for our curent run
         var current_font: Group.FontIndex = .{};
@@ -83,21 +171,56 @@ pub const RunIterator = struct {
         self.shaper.hb_buf.reset();
         self.shaper.hb_buf.setContentType(.unicode);
 
+        // Harfbuzz lets you assign an arbitrary "cluster value" to each
+        // codepoint in a buffer. We use this to determine character width.
+        // Character width is KIND OF BROKEN with terminals because shells
+        // and client applications tend to use wcswidth(3) and friends to
+        // determine width which is broken for unicode graphemes. However,
+        // we need to match it otherwise things are really broken.
+        var cluster: u16 = 0;
+
         // Go through cell by cell and accumulate while we build our run.
         var j: usize = self.i;
-        while (j < self.row.lenCells()) : (j += 1) {
+        while (j < max) : (j += 1) {
             const cell = self.row.getCell(j);
 
-            // Ignore tailing wide spacers, this will get fixed up by the shaper
+            // If we're a spacer, then we ignore it but increase the max cluster
-            if (cell.empty() or cell.attrs.wide_spacer_tail) continue;
+            // size so that the width calculation is correct.
+            if (cell.attrs.wide_spacer_tail) continue;
 
             const style: Style = if (cell.attrs.bold)
                 .bold
             else
                 .regular;
 
-            // Determine the font for this cell
+            // Determine the presentation format for this glyph.
-            const font_idx_opt = try self.shaper.group.indexForCodepoint(alloc, style, cell.char);
+            const presentation: ?Presentation = if (cell.attrs.grapheme) p: {
+                // We only check the FIRST codepoint because I believe the
+                // presentation format must be directly adjacent to the codepoint.
+                var it = self.row.codepointIterator(j);
+                if (it.next()) |cp| {
+                    if (cp == 0xFE0E) break :p Presentation.text;
+                    if (cp == 0xFE0F) break :p Presentation.emoji;
+                }
+
+                break :p null;
+            } else null;
+
+            // Determine the font for this cell. We'll use fallbacks
+            // manually here to try replacement chars and then a space
+            // for unknown glyphs.
+            const font_idx_opt = (try self.shaper.group.indexForCodepoint(
+                alloc,
+                if (cell.empty()) ' ' else cell.char,
+                style,
+                presentation,
+            )) orelse (try self.shaper.group.indexForCodepoint(
+                alloc,
+                0xFFFD,
+                style,
+                .text,
+            )) orelse
+                try self.shaper.group.indexForCodepoint(alloc, ' ', style, .text);
             const font_idx = font_idx_opt.?;
             //log.warn("char={x} idx={}", .{ cell.char, font_idx });
             if (j == self.i) current_font = font_idx;
@@ -106,14 +229,18 @@ pub const RunIterator = struct {
             if (font_idx.int() != current_font.int()) break;
 
             // Continue with our run
-            self.shaper.hb_buf.add(cell.char, @intCast(u32, j));
+            self.shaper.hb_buf.add(cell.char, @intCast(u32, cluster));
+
+            // Increase our cluster value by the width of this cell.
+            cluster += cell.widthLegacy();
 
             // If this cell is part of a grapheme cluster, add all the grapheme
             // data points.
             if (cell.attrs.grapheme) {
                 var it = self.row.codepointIterator(j);
                 while (it.next()) |cp| {
-                    self.shaper.hb_buf.add(cp, @intCast(u32, j));
+                    if (cp == 0xFE0E or cp == 0xFE0F) continue;
+                    self.shaper.hb_buf.add(cp, @intCast(u32, cluster));
                 }
             }
         }
@@ -121,10 +248,15 @@ pub const RunIterator = struct {
         // Finalize our buffer
         self.shaper.hb_buf.guessSegmentProperties();
 
-        // Move our cursor
+        // Move our cursor. Must defer since we use self.i below.
-        self.i = j;
+        defer self.i = j;
 
-        return TextRun{ .font_index = current_font };
+        return TextRun{
+            .offset = @intCast(u16, self.i),
+            .cells = @intCast(u16, j - self.i),
+            .max_cluster = cluster,
+            .font_index = current_font,
+        };
     }
 };
 
@@ -149,6 +281,19 @@ test "run iterator" {
         try testing.expectEqual(@as(usize, 1), count);
     }
 
+    // Spaces should be part of a run
+    {
+        var screen = try terminal.Screen.init(alloc, 3, 10, 0);
+        defer screen.deinit();
+        try screen.testWriteString("ABCD   EFG");
+
+        var shaper = testdata.shaper;
+        var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+        var count: usize = 0;
+        while (try it.next(alloc)) |_| count += 1;
+        try testing.expectEqual(@as(usize, 1), count);
+    }
+
     {
         // Make a screen with some data
         var screen = try terminal.Screen.init(alloc, 3, 5, 0);
@@ -194,7 +339,207 @@ test "shape" {
     while (try it.next(alloc)) |run| {
         count += 1;
         try testing.expectEqual(@as(u32, 3), shaper.hb_buf.getLength());
-        shaper.shape(run);
+        _ = try shaper.shape(run);
+    }
+    try testing.expectEqual(@as(usize, 1), count);
+}
+
+test "shape inconsolata ligs" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var testdata = try testShaper(alloc);
+    defer testdata.deinit();
+
+    {
+        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
+        defer screen.deinit();
+        try screen.testWriteString(">=");
+
+        var shaper = testdata.shaper;
+        var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+        var count: usize = 0;
+        while (try it.next(alloc)) |run| {
+            count += 1;
+
+            const cells = try shaper.shape(run);
+            try testing.expectEqual(@as(usize, 1), cells.len);
+            try testing.expectEqual(@as(u8, 2), cells[0].width);
+        }
+        try testing.expectEqual(@as(usize, 1), count);
+    }
+
+    {
+        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
+        defer screen.deinit();
+        try screen.testWriteString("===");
+
+        var shaper = testdata.shaper;
+        var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+        var count: usize = 0;
+        while (try it.next(alloc)) |run| {
+            count += 1;
+
+            const cells = try shaper.shape(run);
+            try testing.expectEqual(@as(usize, 1), cells.len);
+            try testing.expectEqual(@as(u8, 3), cells[0].width);
+        }
+        try testing.expectEqual(@as(usize, 1), count);
+    }
+}
+
+test "shape emoji width" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var testdata = try testShaper(alloc);
+    defer testdata.deinit();
+
+    {
+        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
+        defer screen.deinit();
+        try screen.testWriteString("👍");
+
+        var shaper = testdata.shaper;
+        var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+        var count: usize = 0;
+        while (try it.next(alloc)) |run| {
+            count += 1;
+
+            const cells = try shaper.shape(run);
+            try testing.expectEqual(@as(usize, 1), cells.len);
+            try testing.expectEqual(@as(u8, 2), cells[0].width);
+        }
+        try testing.expectEqual(@as(usize, 1), count);
+    }
+}
+
+test "shape emoji width long" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var testdata = try testShaper(alloc);
+    defer testdata.deinit();
+
+    var buf: [32]u8 = undefined;
+    var buf_idx: usize = 0;
+    buf_idx += try std.unicode.utf8Encode(0x1F9D4, buf[buf_idx..]); // man: beard
+    buf_idx += try std.unicode.utf8Encode(0x1F3FB, buf[buf_idx..]); // light skin tone (Fitz 1-2)
+    buf_idx += try std.unicode.utf8Encode(0x200D, buf[buf_idx..]); // ZWJ
+    buf_idx += try std.unicode.utf8Encode(0x2642, buf[buf_idx..]); // male sign
+    buf_idx += try std.unicode.utf8Encode(0xFE0F, buf[buf_idx..]); // emoji representation
+
+    // Make a screen with some data
+    var screen = try terminal.Screen.init(alloc, 3, 30, 0);
+    defer screen.deinit();
+    try screen.testWriteString(buf[0..buf_idx]);
+
+    // Get our run iterator
+    var shaper = testdata.shaper;
+    var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+    var count: usize = 0;
+    while (try it.next(alloc)) |run| {
+        count += 1;
+        try testing.expectEqual(@as(u32, 4), shaper.hb_buf.getLength());
+
+        const cells = try shaper.shape(run);
+        try testing.expectEqual(@as(usize, 1), cells.len);
+        try testing.expectEqual(@as(u8, 5), cells[0].width);
+    }
+    try testing.expectEqual(@as(usize, 1), count);
+}
+
+test "shape variation selector VS15" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var testdata = try testShaper(alloc);
+    defer testdata.deinit();
+
+    var buf: [32]u8 = undefined;
+    var buf_idx: usize = 0;
+    buf_idx += try std.unicode.utf8Encode(0x270C, buf[buf_idx..]); // Victory sign (default text)
+    buf_idx += try std.unicode.utf8Encode(0xFE0E, buf[buf_idx..]); // ZWJ to force text
+
+    // Make a screen with some data
+    var screen = try terminal.Screen.init(alloc, 3, 10, 0);
+    defer screen.deinit();
+    try screen.testWriteString(buf[0..buf_idx]);
+
+    // Get our run iterator
+    var shaper = testdata.shaper;
+    var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+    var count: usize = 0;
+    while (try it.next(alloc)) |run| {
+        count += 1;
+        try testing.expectEqual(@as(u32, 1), shaper.hb_buf.getLength());
+
+        const cells = try shaper.shape(run);
+        try testing.expectEqual(@as(usize, 1), cells.len);
+        try testing.expectEqual(@as(u8, 1), cells[0].width);
+    }
+    try testing.expectEqual(@as(usize, 1), count);
+}
+
+test "shape variation selector VS16" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var testdata = try testShaper(alloc);
+    defer testdata.deinit();
+
+    var buf: [32]u8 = undefined;
+    var buf_idx: usize = 0;
+    buf_idx += try std.unicode.utf8Encode(0x270C, buf[buf_idx..]); // Victory sign (default text)
+    buf_idx += try std.unicode.utf8Encode(0xFE0F, buf[buf_idx..]); // ZWJ to force color
+
+    // Make a screen with some data
+    var screen = try terminal.Screen.init(alloc, 3, 10, 0);
+    defer screen.deinit();
+    try screen.testWriteString(buf[0..buf_idx]);
+
+    // Get our run iterator
+    var shaper = testdata.shaper;
+    var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+    var count: usize = 0;
+    while (try it.next(alloc)) |run| {
+        count += 1;
+        try testing.expectEqual(@as(u32, 1), shaper.hb_buf.getLength());
+
+        const cells = try shaper.shape(run);
+        try testing.expectEqual(@as(usize, 1), cells.len);
+
+        // TODO: this should pass, victory sign is width one but
+        // after forcing color it is width 2
+        //try testing.expectEqual(@as(u8, 2), cells[0].width);
+        try testing.expectEqual(@as(u8, 1), cells[0].width);
+    }
+    try testing.expectEqual(@as(usize, 1), count);
+}
+
+test "shape with empty cells in between" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var testdata = try testShaper(alloc);
+    defer testdata.deinit();
+
+    // Make a screen with some data
+    var screen = try terminal.Screen.init(alloc, 3, 30, 0);
+    defer screen.deinit();
+    try screen.testWriteString("A");
+    screen.cursor.x += 5;
+    try screen.testWriteString("B");
+
+    // Get our run iterator
+    var shaper = testdata.shaper;
+    var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+    var count: usize = 0;
+    while (try it.next(alloc)) |run| {
+        count += 1;
+
+        const cells = try shaper.shape(run);
+        try testing.expectEqual(@as(usize, 7), cells.len);
     }
     try testing.expectEqual(@as(usize, 1), count);
 }
@@ -204,11 +549,13 @@ const TestShaper = struct {
     shaper: Shaper,
     cache: *GroupCache,
     lib: Library,
+    cell_buf: []Cell,
 
     pub fn deinit(self: *TestShaper) void {
         self.shaper.deinit();
         self.cache.deinit(self.alloc);
         self.alloc.destroy(self.cache);
+        self.alloc.free(self.cell_buf);
         self.lib.deinit();
     }
 };
@@ -217,6 +564,7 @@ const TestShaper = struct {
 fn testShaper(alloc: Allocator) !TestShaper {
     const testFont = @import("test.zig").fontRegular;
     const testEmoji = @import("test.zig").fontEmoji;
+    const testEmojiText = @import("test.zig").fontEmojiText;
 
     var lib = try Library.init();
     errdefer lib.deinit();
@@ -229,8 +577,12 @@ fn testShaper(alloc: Allocator) !TestShaper {
     // Setup group
     try cache_ptr.group.addFace(alloc, .regular, try Face.init(lib, testFont, .{ .points = 12 }));
     try cache_ptr.group.addFace(alloc, .regular, try Face.init(lib, testEmoji, .{ .points = 12 }));
+    try cache_ptr.group.addFace(alloc, .regular, try Face.init(lib, testEmojiText, .{ .points = 12 }));
 
-    var shaper = try init(cache_ptr);
+    var cell_buf = try alloc.alloc(Cell, 80);
+    errdefer alloc.free(cell_buf);
+
+    var shaper = try init(cache_ptr, cell_buf);
     errdefer shaper.deinit();
 
     return TestShaper{
@@ -238,5 +590,6 @@ fn testShaper(alloc: Allocator) !TestShaper {
         .shaper = shaper,
         .cache = cache_ptr,
         .lib = lib,
+        .cell_buf = cell_buf,
     };
 }

src/font/main.zig

@@ -8,13 +8,19 @@ pub const Library = @import("Library.zig");
 pub const Shaper = @import("Shaper.zig");
 
 /// The styles that a family can take.
-pub const Style = enum(u2) {
+pub const Style = enum(u3) {
     regular = 0,
     bold = 1,
     italic = 2,
     bold_italic = 3,
 };
 
+/// The presentation for a an emoji.
+pub const Presentation = enum(u1) {
+    text = 0, // U+FE0E
+    emoji = 1, // U+FEOF
+};
+
 /// Font metrics useful for things such as grid calculation.
 pub const Metrics = struct {
     /// The width and height of a monospace cell.

src/font/test.zig

@@ -1,3 +1,4 @@
 pub const fontRegular = @embedFile("res/Inconsolata-Regular.ttf");
 pub const fontBold = @embedFile("res/Inconsolata-Bold.ttf");
 pub const fontEmoji = @embedFile("res/NotoColorEmoji.ttf");
+pub const fontEmojiText = @embedFile("res/NotoEmoji-Regular.ttf");

src/terminal/Screen.zig

@@ -181,6 +181,39 @@ pub const Cell = struct {
         return self.char == 0;
     }
 
+    /// The width of the cell.
+    ///
+    /// This uses the legacy calculation of a per-codepoint width calculation
+    /// to determine the width. This legacy calculation is incorrect because
+    /// it doesn't take into account multi-codepoint graphemes.
+    ///
+    /// The goal of this function is to match the expectation of shells
+    /// that aren't grapheme aware (at the time of writing this comment: none
+    /// are grapheme aware). This means it should match wcswidth.
+    pub fn widthLegacy(self: Cell) u16 {
+        // Wide is always 2
+        if (self.attrs.wide) return 2;
+
+        // Wide spacers are always 0 because their width is accounted for
+        // in the wide char.
+        if (self.attrs.wide_spacer_tail or self.attrs.wide_spacer_head) return 0;
+
+        return 1;
+    }
+
+    test "widthLegacy" {
+        const testing = std.testing;
+
+        var c: Cell = .{};
+        try testing.expectEqual(@as(u16, 1), c.widthLegacy());
+
+        c = .{ .attrs = .{ .wide = true } };
+        try testing.expectEqual(@as(u16, 2), c.widthLegacy());
+
+        c = .{ .attrs = .{ .wide_spacer_tail = true } };
+        try testing.expectEqual(@as(u16, 0), c.widthLegacy());
+    }
+
     test {
         // We use this test to ensure we always get the right size of the attrs
         // const cell: Cell = .{ .char = 0 };
@@ -1497,6 +1530,8 @@ pub fn testWriteString(self: *Screen, text: []const u8) !void {
         // Get our row
         var row = self.getRow(.{ .active = y });
 
+        // NOTE: graphemes are currently disabled
+        if (false) {
             // If we have a previous cell, we check if we're part of a grapheme.
             if (grapheme.cell) |prev_cell| {
                 const grapheme_break = brk: {
@@ -1523,6 +1558,21 @@ pub fn testWriteString(self: *Screen, text: []const u8) !void {
                     continue;
                 }
             }
+        }
+
+        const width = utf8proc.charwidth(c);
+        //log.warn("c={x} width={}", .{ c, width });
+
+        // Zero-width are attached as grapheme data.
+        // NOTE: if/when grapheme clustering is ever enabled (above) this
+        // is not necessary
+        if (width == 0) {
+            if (grapheme.cell != null) {
+                try row.attachGrapheme(grapheme.x, c);
+            }
+
+            continue;
+        }
 
         // If we're writing past the end, we need to soft wrap.
         if (x == self.cols) {
@@ -1537,7 +1587,6 @@ pub fn testWriteString(self: *Screen, text: []const u8) !void {
         }
 
         // If our character is double-width, handle it.
-        const width = utf8proc.charwidth(c);
         assert(width == 1 or width == 2);
         switch (width) {
             1 => {
@@ -1768,9 +1817,40 @@ test "Screen: write graphemes" {
     buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain
     buf_idx += try std.unicode.utf8Encode(0x1F3FD, buf[buf_idx..]); // Medium skin tone
 
+    // Note the assertions below are NOT the correct way to handle graphemes
+    // in general, but they're "correct" for historical purposes for terminals.
+    // For terminals, all double-wide codepoints are counted as part of the
+    // width.
+
+    try s.testWriteString(buf[0..buf_idx]);
+    try testing.expect(s.rowsWritten() == 2);
+    try testing.expectEqual(@as(usize, 2), s.cursor.x);
+}
+
+test "Screen: write long emoji" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var s = try init(alloc, 5, 30, 0);
+    defer s.deinit();
+
+    // Sanity check that our test helpers work
+    var buf: [32]u8 = undefined;
+    var buf_idx: usize = 0;
+    buf_idx += try std.unicode.utf8Encode(0x1F9D4, buf[buf_idx..]); // man: beard
+    buf_idx += try std.unicode.utf8Encode(0x1F3FB, buf[buf_idx..]); // light skin tone (Fitz 1-2)
+    buf_idx += try std.unicode.utf8Encode(0x200D, buf[buf_idx..]); // ZWJ
+    buf_idx += try std.unicode.utf8Encode(0x2642, buf[buf_idx..]); // male sign
+    buf_idx += try std.unicode.utf8Encode(0xFE0F, buf[buf_idx..]); // emoji representation
+
+    // Note the assertions below are NOT the correct way to handle graphemes
+    // in general, but they're "correct" for historical purposes for terminals.
+    // For terminals, all double-wide codepoints are counted as part of the
+    // width.
+
     try s.testWriteString(buf[0..buf_idx]);
     try testing.expect(s.rowsWritten() == 1);
-    try testing.expectEqual(@as(usize, 4), s.cursor.x);
+    try testing.expectEqual(@as(usize, 5), s.cursor.x);
 }
 
 test "Screen: scrolling" {