ghostty/src/font/Shaper.zig

//! This struct handles text shaping.
const Shaper = @This();

const std = @import("std");
const assert = std.debug.assert;
const Allocator = std.mem.Allocator;
const harfbuzz = @import("harfbuzz");
const trace = @import("tracy").trace;
const font = @import("main.zig");
const Face = @import("main.zig").Face;
const DeferredFace = @import("main.zig").DeferredFace;
const Group = @import("main.zig").Group;
const GroupCache = @import("main.zig").GroupCache;
const Library = @import("main.zig").Library;
const Style = @import("main.zig").Style;
const Presentation = @import("main.zig").Presentation;
const terminal = @import("../terminal/main.zig");

const log = std.log.scoped(.font_shaper);

/// The buffer used for text shaping. We reuse it across multiple shaping
/// calls to prevent allocations.
hb_buf: harfbuzz.Buffer,

/// The shared memory used for shaping results.
cell_buf: []Cell,

/// The cell_buf argument is the buffer to use for storing shaped results.
/// This should be at least the number of columns in the terminal.
pub fn init(cell_buf: []Cell) !Shaper {
    return Shaper{
        .hb_buf = try harfbuzz.Buffer.create(),
        .cell_buf = cell_buf,
    };
}

pub fn deinit(self: *Shaper) void {
    self.hb_buf.destroy();
}

/// Returns an iterator that returns one text run at a time for the
/// given terminal row. Note that text runs are are only valid one at a time
/// for a Shaper struct since they share state.
pub fn runIterator(self: *Shaper, group: *GroupCache, row: terminal.Screen.Row) RunIterator {
    return .{ .shaper = self, .group = group, .row = row };
}

/// Shape the given text run. The text run must be the immediately previous
/// text run that was iterated since the text run does share state with the
/// Shaper struct.
///
/// The return value is only valid until the next shape call is called.
///
/// If there is not enough space in the cell buffer, an error is returned.
pub fn shape(self: *Shaper, run: TextRun) ![]Cell {
    const tracy = trace(@src());
    defer tracy.end();

    // We only do shaping if the font is not a special-case. For special-case
    // fonts, the codepoint == glyph_index so we don't need to run any shaping.
    if (run.font_index.special() == null) {
        // TODO: we do not want to hardcode these
        const hb_feats = &[_]harfbuzz.Feature{
            harfbuzz.Feature.fromString("dlig").?,
            harfbuzz.Feature.fromString("liga").?,
        };

        const face = try run.group.group.faceFromIndex(run.font_index);
        harfbuzz.shape(face.hb_font, self.hb_buf, hb_feats);
    }

    // If our buffer is empty, we short-circuit the rest of the work
    // return nothing.
    if (self.hb_buf.getLength() == 0) return self.cell_buf[0..0];
    const info = self.hb_buf.getGlyphInfos();
    const pos = self.hb_buf.getGlyphPositions() orelse return error.HarfbuzzFailed;

    // This is perhaps not true somewhere, but we currently assume it is true.
    // If it isn't true, I'd like to catch it and learn more.
    assert(info.len == pos.len);

    // Convert all our info/pos to cells and set it.
    if (info.len > self.cell_buf.len) return error.OutOfMemory;
    //log.warn("info={} pos={} run={}", .{ info.len, pos.len, run });

    for (info) |v, i| {
        self.cell_buf[i] = .{
            .x = @intCast(u16, v.cluster),
            .glyph_index = v.codepoint,
        };

        //log.warn("i={} info={} pos={} cell={}", .{ i, v, pos[i], self.cell_buf[i] });
    }

    return self.cell_buf[0..info.len];
}

pub const Cell = struct {
    /// The column that this cell occupies. Since a set of shaper cells is
    /// always on the same line, only the X is stored. It is expected the
    /// caller has access to the original screen cell.
    x: u16,

    /// The glyph index for this cell. The font index to use alongside
    /// this cell is available in the text run.
    glyph_index: u32,
};

/// A single text run. A text run is only valid for one Shaper and
/// until the next run is created.
pub const TextRun = struct {
    /// The offset in the row where this run started
    offset: u16,

    /// The total number of cells produced by this run.
    cells: u16,

    /// The font group that built this run.
    group: *GroupCache,

    /// The font index to use for the glyphs of this run.
    font_index: Group.FontIndex,
};

pub const RunIterator = struct {
    shaper: *Shaper,
    group: *GroupCache,
    row: terminal.Screen.Row,
    i: usize = 0,

    pub fn next(self: *RunIterator, alloc: Allocator) !?TextRun {
        const tracy = trace(@src());
        defer tracy.end();

        // Trim the right side of a row that might be empty
        const max: usize = max: {
            var j: usize = self.row.lenCells();
            while (j > 0) : (j -= 1) if (!self.row.getCell(j - 1).empty()) break;
            break :max j;
        };

        // We're over at the max
        if (self.i >= max) return null;

        // Track the font for our curent run
        var current_font: Group.FontIndex = .{};

        // Reset the buffer for our current run
        self.shaper.hb_buf.reset();
        self.shaper.hb_buf.setContentType(.unicode);

        // Go through cell by cell and accumulate while we build our run.
        var j: usize = self.i;
        while (j < max) : (j += 1) {
            const cluster = j;
            const cell = self.row.getCell(j);

            // If we're a spacer, then we ignore it
            if (cell.attrs.wide_spacer_tail) continue;

            const style: Style = if (cell.attrs.bold)
                .bold
            else
                .regular;

            // Determine the presentation format for this glyph.
            const presentation: ?Presentation = if (cell.attrs.grapheme) p: {
                // We only check the FIRST codepoint because I believe the
                // presentation format must be directly adjacent to the codepoint.
                var it = self.row.codepointIterator(j);
                if (it.next()) |cp| {
                    if (cp == 0xFE0E) break :p Presentation.text;
                    if (cp == 0xFE0F) break :p Presentation.emoji;
                }

                break :p null;
            } else null;

            // Determine the font for this cell. We'll use fallbacks
            // manually here to try replacement chars and then a space
            // for unknown glyphs.
            const font_idx_opt = (try self.group.indexForCodepoint(
                alloc,
                if (cell.empty() or cell.char == 0) ' ' else cell.char,
                style,
                presentation,
            )) orelse (try self.group.indexForCodepoint(
                alloc,
                0xFFFD,
                style,
                .text,
            )) orelse
                try self.group.indexForCodepoint(alloc, ' ', style, .text);
            const font_idx = font_idx_opt.?;
            //log.warn("char={x} idx={}", .{ cell.char, font_idx });
            if (j == self.i) current_font = font_idx;

            // If our fonts are not equal, then we're done with our run.
            if (font_idx.int() != current_font.int()) break;

            // Continue with our run
            self.shaper.hb_buf.add(cell.char, @intCast(u32, cluster));

            // If this cell is part of a grapheme cluster, add all the grapheme
            // data points.
            if (cell.attrs.grapheme) {
                var it = self.row.codepointIterator(j);
                while (it.next()) |cp| {
                    if (cp == 0xFE0E or cp == 0xFE0F) continue;
                    self.shaper.hb_buf.add(cp, @intCast(u32, cluster));
                }
            }
        }

        // Finalize our buffer
        self.shaper.hb_buf.guessSegmentProperties();

        // Move our cursor. Must defer since we use self.i below.
        defer self.i = j;

        return TextRun{
            .offset = @intCast(u16, self.i),
            .cells = @intCast(u16, j - self.i),
            .group = self.group,
            .font_index = current_font,
        };
    }
};

test "run iterator" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    {
        // Make a screen with some data
        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
        defer screen.deinit();
        try screen.testWriteString("ABCD");

        // Get our run iterator
        var shaper = testdata.shaper;
        var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
        var count: usize = 0;
        while (try it.next(alloc)) |_| count += 1;
        try testing.expectEqual(@as(usize, 1), count);
    }

    // Spaces should be part of a run
    {
        var screen = try terminal.Screen.init(alloc, 3, 10, 0);
        defer screen.deinit();
        try screen.testWriteString("ABCD   EFG");

        var shaper = testdata.shaper;
        var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
        var count: usize = 0;
        while (try it.next(alloc)) |_| count += 1;
        try testing.expectEqual(@as(usize, 1), count);
    }

    {
        // Make a screen with some data
        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
        defer screen.deinit();
        try screen.testWriteString("A😃D");

        // Get our run iterator
        var shaper = testdata.shaper;
        var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
        var count: usize = 0;
        while (try it.next(alloc)) |_| {
            count += 1;

            // All runs should be exactly length 1
            try testing.expectEqual(@as(u32, 1), shaper.hb_buf.getLength());
        }
        try testing.expectEqual(@as(usize, 3), count);
    }
}

test "run iterator: empty cells with background set" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    {
        // Make a screen with some data
        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
        defer screen.deinit();
        try screen.testWriteString("A");

        // Get our first row
        const row = screen.getRow(.{ .active = 0 });
        row.getCellPtr(1).bg = try terminal.color.Name.cyan.default();
        row.getCellPtr(1).attrs.has_bg = true;
        row.getCellPtr(2).fg = try terminal.color.Name.yellow.default();
        row.getCellPtr(2).attrs.has_fg = true;

        // Get our run iterator
        var shaper = testdata.shaper;
        var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
        var count: usize = 0;
        while (try it.next(alloc)) |run| {
            count += 1;

            // The run should have length 3 because of the two background
            // cells.
            try testing.expectEqual(@as(u32, 3), shaper.hb_buf.getLength());
            const cells = try shaper.shape(run);
            try testing.expectEqual(@as(usize, 3), cells.len);
        }
        try testing.expectEqual(@as(usize, 1), count);
    }
}
test "shape" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    var buf: [32]u8 = undefined;
    var buf_idx: usize = 0;
    buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain
    buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain
    buf_idx += try std.unicode.utf8Encode(0x1F3FD, buf[buf_idx..]); // Medium skin tone

    // Make a screen with some data
    var screen = try terminal.Screen.init(alloc, 3, 10, 0);
    defer screen.deinit();
    try screen.testWriteString(buf[0..buf_idx]);

    // Get our run iterator
    var shaper = testdata.shaper;
    var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
    var count: usize = 0;
    while (try it.next(alloc)) |run| {
        count += 1;
        try testing.expectEqual(@as(u32, 3), shaper.hb_buf.getLength());
        _ = try shaper.shape(run);
    }
    try testing.expectEqual(@as(usize, 1), count);
}

test "shape inconsolata ligs" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    {
        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
        defer screen.deinit();
        try screen.testWriteString(">=");

        var shaper = testdata.shaper;
        var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
        var count: usize = 0;
        while (try it.next(alloc)) |run| {
            count += 1;

            const cells = try shaper.shape(run);
            try testing.expectEqual(@as(usize, 1), cells.len);
        }
        try testing.expectEqual(@as(usize, 1), count);
    }

    {
        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
        defer screen.deinit();
        try screen.testWriteString("===");

        var shaper = testdata.shaper;
        var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
        var count: usize = 0;
        while (try it.next(alloc)) |run| {
            count += 1;

            const cells = try shaper.shape(run);
            try testing.expectEqual(@as(usize, 1), cells.len);
        }
        try testing.expectEqual(@as(usize, 1), count);
    }
}

test "shape emoji width" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    {
        var screen = try terminal.Screen.init(alloc, 3, 5, 0);
        defer screen.deinit();
        try screen.testWriteString("👍");

        var shaper = testdata.shaper;
        var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
        var count: usize = 0;
        while (try it.next(alloc)) |run| {
            count += 1;

            const cells = try shaper.shape(run);
            try testing.expectEqual(@as(usize, 1), cells.len);
        }
        try testing.expectEqual(@as(usize, 1), count);
    }
}

test "shape emoji width long" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    var buf: [32]u8 = undefined;
    var buf_idx: usize = 0;
    buf_idx += try std.unicode.utf8Encode(0x1F9D4, buf[buf_idx..]); // man: beard
    buf_idx += try std.unicode.utf8Encode(0x1F3FB, buf[buf_idx..]); // light skin tone (Fitz 1-2)
    buf_idx += try std.unicode.utf8Encode(0x200D, buf[buf_idx..]); // ZWJ
    buf_idx += try std.unicode.utf8Encode(0x2642, buf[buf_idx..]); // male sign
    buf_idx += try std.unicode.utf8Encode(0xFE0F, buf[buf_idx..]); // emoji representation

    // Make a screen with some data
    var screen = try terminal.Screen.init(alloc, 3, 30, 0);
    defer screen.deinit();
    try screen.testWriteString(buf[0..buf_idx]);

    // Get our run iterator
    var shaper = testdata.shaper;
    var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
    var count: usize = 0;
    while (try it.next(alloc)) |run| {
        count += 1;
        try testing.expectEqual(@as(u32, 4), shaper.hb_buf.getLength());

        const cells = try shaper.shape(run);
        try testing.expectEqual(@as(usize, 1), cells.len);
    }
    try testing.expectEqual(@as(usize, 1), count);
}

test "shape variation selector VS15" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    var buf: [32]u8 = undefined;
    var buf_idx: usize = 0;
    buf_idx += try std.unicode.utf8Encode(0x270C, buf[buf_idx..]); // Victory sign (default text)
    buf_idx += try std.unicode.utf8Encode(0xFE0E, buf[buf_idx..]); // ZWJ to force text

    // Make a screen with some data
    var screen = try terminal.Screen.init(alloc, 3, 10, 0);
    defer screen.deinit();
    try screen.testWriteString(buf[0..buf_idx]);

    // Get our run iterator
    var shaper = testdata.shaper;
    var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
    var count: usize = 0;
    while (try it.next(alloc)) |run| {
        count += 1;
        try testing.expectEqual(@as(u32, 1), shaper.hb_buf.getLength());

        const cells = try shaper.shape(run);
        try testing.expectEqual(@as(usize, 1), cells.len);
    }
    try testing.expectEqual(@as(usize, 1), count);
}

test "shape variation selector VS16" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    var buf: [32]u8 = undefined;
    var buf_idx: usize = 0;
    buf_idx += try std.unicode.utf8Encode(0x270C, buf[buf_idx..]); // Victory sign (default text)
    buf_idx += try std.unicode.utf8Encode(0xFE0F, buf[buf_idx..]); // ZWJ to force color

    // Make a screen with some data
    var screen = try terminal.Screen.init(alloc, 3, 10, 0);
    defer screen.deinit();
    try screen.testWriteString(buf[0..buf_idx]);

    // Get our run iterator
    var shaper = testdata.shaper;
    var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
    var count: usize = 0;
    while (try it.next(alloc)) |run| {
        count += 1;
        try testing.expectEqual(@as(u32, 1), shaper.hb_buf.getLength());

        const cells = try shaper.shape(run);
        try testing.expectEqual(@as(usize, 1), cells.len);
    }
    try testing.expectEqual(@as(usize, 1), count);
}

test "shape with empty cells in between" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    // Make a screen with some data
    var screen = try terminal.Screen.init(alloc, 3, 30, 0);
    defer screen.deinit();
    try screen.testWriteString("A");
    screen.cursor.x += 5;
    try screen.testWriteString("B");

    // Get our run iterator
    var shaper = testdata.shaper;
    var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
    var count: usize = 0;
    while (try it.next(alloc)) |run| {
        count += 1;

        const cells = try shaper.shape(run);
        try testing.expectEqual(@as(usize, 7), cells.len);
    }
    try testing.expectEqual(@as(usize, 1), count);
}

test "shape Chinese characters" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    var buf: [32]u8 = undefined;
    var buf_idx: usize = 0;
    buf_idx += try std.unicode.utf8Encode('n', buf[buf_idx..]); // Combining
    buf_idx += try std.unicode.utf8Encode(0x0308, buf[buf_idx..]); // Combining
    buf_idx += try std.unicode.utf8Encode(0x0308, buf[buf_idx..]);
    buf_idx += try std.unicode.utf8Encode('a', buf[buf_idx..]);

    // Make a screen with some data
    var screen = try terminal.Screen.init(alloc, 3, 30, 0);
    defer screen.deinit();
    try screen.testWriteString(buf[0..buf_idx]);

    // Get our run iterator
    var shaper = testdata.shaper;
    var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
    var count: usize = 0;
    while (try it.next(alloc)) |run| {
        count += 1;

        const cells = try shaper.shape(run);
        try testing.expectEqual(@as(usize, 4), cells.len);
        try testing.expectEqual(@as(u16, 0), cells[0].x);
        try testing.expectEqual(@as(u16, 0), cells[1].x);
        try testing.expectEqual(@as(u16, 0), cells[2].x);
        try testing.expectEqual(@as(u16, 1), cells[3].x);
    }
    try testing.expectEqual(@as(usize, 1), count);
}

test "shape box glyphs" {
    const testing = std.testing;
    const alloc = testing.allocator;

    var testdata = try testShaper(alloc);
    defer testdata.deinit();

    // Setup the box font
    testdata.cache.group.sprite = font.sprite.Face{
        .width = 18,
        .height = 36,
        .thickness = 2,
    };

    var buf: [32]u8 = undefined;
    var buf_idx: usize = 0;
    buf_idx += try std.unicode.utf8Encode(0x2500, buf[buf_idx..]); // horiz line
    buf_idx += try std.unicode.utf8Encode(0x2501, buf[buf_idx..]); //

    // Make a screen with some data
    var screen = try terminal.Screen.init(alloc, 3, 10, 0);
    defer screen.deinit();
    try screen.testWriteString(buf[0..buf_idx]);

    // Get our run iterator
    var shaper = testdata.shaper;
    var it = shaper.runIterator(testdata.cache, screen.getRow(.{ .screen = 0 }));
    var count: usize = 0;
    while (try it.next(alloc)) |run| {
        count += 1;
        try testing.expectEqual(@as(u32, 2), shaper.hb_buf.getLength());
        const cells = try shaper.shape(run);
        try testing.expectEqual(@as(usize, 2), cells.len);
        try testing.expectEqual(@as(u32, 0x2500), cells[0].glyph_index);
        try testing.expectEqual(@as(u16, 0), cells[0].x);
        try testing.expectEqual(@as(u32, 0x2501), cells[1].glyph_index);
        try testing.expectEqual(@as(u16, 1), cells[1].x);
    }
    try testing.expectEqual(@as(usize, 1), count);
}

const TestShaper = struct {
    alloc: Allocator,
    shaper: Shaper,
    cache: *GroupCache,
    lib: Library,
    cell_buf: []Cell,

    pub fn deinit(self: *TestShaper) void {
        self.shaper.deinit();
        self.cache.deinit(self.alloc);
        self.alloc.destroy(self.cache);
        self.alloc.free(self.cell_buf);
        self.lib.deinit();
    }
};

/// Helper to return a fully initialized shaper.
fn testShaper(alloc: Allocator) !TestShaper {
    const testFont = @import("test.zig").fontRegular;
    const testEmoji = @import("test.zig").fontEmoji;
    const testEmojiText = @import("test.zig").fontEmojiText;

    var lib = try Library.init();
    errdefer lib.deinit();

    var cache_ptr = try alloc.create(GroupCache);
    errdefer alloc.destroy(cache_ptr);
    cache_ptr.* = try GroupCache.init(alloc, try Group.init(
        alloc,
        lib,
        .{ .points = 12 },
    ));
    errdefer cache_ptr.*.deinit(alloc);

    // Setup group
    try cache_ptr.group.addFace(alloc, .regular, DeferredFace.initLoaded(try Face.init(lib, testFont, .{ .points = 12 })));
    try cache_ptr.group.addFace(alloc, .regular, DeferredFace.initLoaded(try Face.init(lib, testEmoji, .{ .points = 12 })));
    try cache_ptr.group.addFace(alloc, .regular, DeferredFace.initLoaded(try Face.init(lib, testEmojiText, .{ .points = 12 })));

    var cell_buf = try alloc.alloc(Cell, 80);
    errdefer alloc.free(cell_buf);

    var shaper = try init(cell_buf);
    errdefer shaper.deinit();

    return TestShaper{
        .alloc = alloc,
        .shaper = shaper,
        .cache = cache_ptr,
        .lib = lib,
        .cell_buf = cell_buf,
    };
}