fonts are presentation format aware (text vs emoji)

src/Grid.zig

@@ -166,6 +166,11 @@ pub fn init(
             .regular,
             try font.Face.init(font_lib, face_emoji_ttf, font_size),
         );
+        try group.addFace(
+            alloc,
+            .regular,
+            try font.Face.init(font_lib, face_emoji_text_ttf, font_size),
+        );
 
         break :group group;
     });
@@ -830,3 +835,4 @@ test "GridSize update rounding" {
 const face_ttf = @embedFile("font/res/FiraCode-Regular.ttf");
 const face_bold_ttf = @embedFile("font/res/FiraCode-Bold.ttf");
 const face_emoji_ttf = @embedFile("font/res/NotoColorEmoji.ttf");
+const face_emoji_text_ttf = @embedFile("font/res/NotoEmoji-Regular.ttf");

src/font/Face.zig

@@ -15,6 +15,7 @@ const Allocator = std.mem.Allocator;
 const Atlas = @import("../Atlas.zig");
 const Glyph = @import("main.zig").Glyph;
 const Library = @import("main.zig").Library;
+const Presentation = @import("main.zig").Presentation;
 const convert = @import("convert.zig");
 
 const log = std.log.scoped(.font_face);
@@ -25,6 +26,10 @@ face: freetype.Face,
 /// Harfbuzz font corresponding to this face.
 hb_font: harfbuzz.Font,
 
+/// The presentation for this font. This is a heuristic since fonts don't have
+/// a way to declare this. We just assume a font with color is an emoji font.
+presentation: Presentation,
+
 /// If a DPI can't be calculated, this DPI is used. This is probably
 /// wrong on modern devices so it is highly recommended you get the DPI
 /// using whatever platform method you can.
@@ -56,7 +61,11 @@ pub fn init(lib: Library, source: [:0]const u8, size: DesiredSize) !Face {
     const hb_font = try harfbuzz.freetype.createFont(face.handle);
     errdefer hb_font.destroy();
 
-    return Face{ .face = face, .hb_font = hb_font };
+    return Face{
+        .face = face,
+        .hb_font = hb_font,
+        .presentation = if (face.hasColor()) .emoji else .text,
+    };
 }
 
 pub fn deinit(self: *Face) void {
@@ -241,6 +250,8 @@ test {
     var font = try init(lib, testFont, .{ .points = 12 });
     defer font.deinit();
 
+    try testing.expectEqual(Presentation.text, font.presentation);
+
     // Generate all visible ASCII
     var i: u8 = 32;
     while (i < 127) : (i += 1) {
@@ -261,6 +272,8 @@ test "color emoji" {
     var font = try init(lib, testFont, .{ .points = 12 });
     defer font.deinit();
 
+    try testing.expectEqual(Presentation.emoji, font.presentation);
+
     _ = try font.renderGlyph(alloc, &atlas, font.glyphIndex('🥸').?);
 }
 

src/font/Group.zig

@@ -15,6 +15,7 @@ const Face = @import("main.zig").Face;
 const Library = @import("main.zig").Library;
 const Glyph = @import("main.zig").Glyph;
 const Style = @import("main.zig").Style;
+const Presentation = @import("main.zig").Presentation;
 
 const log = std.log.scoped(.font_group);
 
@@ -86,19 +87,34 @@ pub const FontIndex = packed struct {
 /// The font index is valid as long as font faces aren't removed. This
 /// isn't cached; it is expected that downstream users handle caching if
 /// that is important.
-pub fn indexForCodepoint(self: Group, style: Style, cp: u32) ?FontIndex {
+///
+/// Optionally, a presentation format can be specified. This presentation
+/// format will be preferred but if it can't be found in this format,
+/// any text format will be accepted. If presentation is null, any presentation
+/// is allowed. This func will NOT determine the default presentation for
+/// a code point.
+pub fn indexForCodepoint(
+    self: Group,
+    cp: u32,
+    style: Style,
+    p: ?Presentation,
+) ?FontIndex {
     // If we can find the exact value, then return that.
-    if (self.indexForCodepointExact(style, cp)) |value| return value;
+    if (self.indexForCodepointExact(cp, style, p)) |value| return value;
 
     // If this is already regular, we're done falling back.
-    if (style == .regular) return null;
+    if (style == .regular and p == null) return null;
 
     // For non-regular fonts, we fall back to regular.
-    return self.indexForCodepointExact(.regular, cp);
+    return self.indexForCodepointExact(cp, .regular, null);
 }
 
-fn indexForCodepointExact(self: Group, style: Style, cp: u32) ?FontIndex {
+fn indexForCodepointExact(self: Group, cp: u32, style: Style, p: ?Presentation) ?FontIndex {
     for (self.faces.get(style).items) |face, i| {
+        // If the presentation is null, we allow the first presentation we
+        // can find. Otherwise, we check for the specific one requested.
+        if (p != null and face.presentation != p.?) continue;
+
         if (face.glyphIndex(cp) != null) {
             return FontIndex{
                 .style = style,
@@ -143,6 +159,7 @@ test {
     const alloc = testing.allocator;
     const testFont = @import("test.zig").fontRegular;
     const testEmoji = @import("test.zig").fontEmoji;
+    const testEmojiText = @import("test.zig").fontEmojiText;
 
     var atlas_greyscale = try Atlas.init(alloc, 512, .greyscale);
     defer atlas_greyscale.deinit(alloc);
@@ -155,11 +172,12 @@ test {
 
     try group.addFace(alloc, .regular, try Face.init(lib, testFont, .{ .points = 12 }));
     try group.addFace(alloc, .regular, try Face.init(lib, testEmoji, .{ .points = 12 }));
+    try group.addFace(alloc, .regular, try Face.init(lib, testEmojiText, .{ .points = 12 }));
 
     // Should find all visible ASCII
     var i: u32 = 32;
     while (i < 127) : (i += 1) {
-        const idx = group.indexForCodepoint(.regular, i).?;
+        const idx = group.indexForCodepoint(i, .regular, null).?;
         try testing.expectEqual(Style.regular, idx.style);
         try testing.expectEqual(@as(FontIndex.IndexInt, 0), idx.idx);
 
@@ -176,7 +194,19 @@ test {
 
     // Try emoji
     {
-        const idx = group.indexForCodepoint(.regular, '🥸').?;
+        const idx = group.indexForCodepoint('🥸', .regular, null).?;
+        try testing.expectEqual(Style.regular, idx.style);
+        try testing.expectEqual(@as(FontIndex.IndexInt, 1), idx.idx);
+    }
+
+    // Try text emoji
+    {
+        const idx = group.indexForCodepoint(0x270C, .regular, .text).?;
+        try testing.expectEqual(Style.regular, idx.style);
+        try testing.expectEqual(@as(FontIndex.IndexInt, 2), idx.idx);
+    }
+    {
+        const idx = group.indexForCodepoint(0x270C, .regular, .emoji).?;
         try testing.expectEqual(Style.regular, idx.style);
         try testing.expectEqual(@as(FontIndex.IndexInt, 1), idx.idx);
     }

src/font/GroupCache.zig

@@ -12,6 +12,7 @@ const Glyph = @import("main.zig").Glyph;
 const Style = @import("main.zig").Style;
 const Group = @import("main.zig").Group;
 const Metrics = @import("main.zig").Metrics;
+const Presentation = @import("main.zig").Presentation;
 
 const log = std.log.scoped(.font_groupcache);
 
@@ -34,6 +35,7 @@ atlas_color: Atlas,
 const CodepointKey = struct {
     style: Style,
     codepoint: u32,
+    presentation: ?Presentation,
 };
 
 const GlyphKey = struct {
@@ -90,7 +92,7 @@ pub fn metrics(self: *GroupCache, alloc: Allocator) !Metrics {
         var cell_width: f32 = 0;
         var i: u32 = 32;
         while (i <= 126) : (i += 1) {
-            const index = (try self.indexForCodepoint(alloc, .regular, i)).?;
+            const index = (try self.indexForCodepoint(alloc, i, .regular, .text)).?;
             const face = self.group.faceFromIndex(index);
             const glyph_index = face.glyphIndex(i).?;
             const glyph = try self.renderGlyph(alloc, index, glyph_index);
@@ -106,7 +108,7 @@ pub fn metrics(self: *GroupCache, alloc: Allocator) !Metrics {
     // '_' which should live at the bottom of a cell.
     const cell_height: f32 = cell_height: {
         // Get the '_' char for height
-        const index = (try self.indexForCodepoint(alloc, .regular, '_')).?;
+        const index = (try self.indexForCodepoint(alloc, '_', .regular, .text)).?;
         const face = self.group.faceFromIndex(index);
         const glyph_index = face.glyphIndex('_').?;
         const glyph = try self.renderGlyph(alloc, index, glyph_index);
@@ -142,15 +144,21 @@ pub fn metrics(self: *GroupCache, alloc: Allocator) !Metrics {
 }
 
 /// Get the font index for a given codepoint. This is cached.
-pub fn indexForCodepoint(self: *GroupCache, alloc: Allocator, style: Style, cp: u32) !?Group.FontIndex {
-    const key: CodepointKey = .{ .style = style, .codepoint = cp };
+pub fn indexForCodepoint(
+    self: *GroupCache,
+    alloc: Allocator,
+    cp: u32,
+    style: Style,
+    p: ?Presentation,
+) !?Group.FontIndex {
+    const key: CodepointKey = .{ .style = style, .codepoint = cp, .presentation = p };
     const gop = try self.codepoints.getOrPut(alloc, key);
 
     // If it is in the cache, use it.
     if (gop.found_existing) return gop.value_ptr.*;
 
     // Load a value and cache it. This even caches negative matches.
-    const value = self.group.indexForCodepoint(style, cp);
+    const value = self.group.indexForCodepoint(cp, style, null);
     gop.value_ptr.* = value;
     return value;
 }
@@ -219,7 +227,7 @@ test {
     // Visible ASCII. Do it twice to verify cache.
     var i: u32 = 32;
     while (i < 127) : (i += 1) {
-        const idx = (try cache.indexForCodepoint(alloc, .regular, i)).?;
+        const idx = (try cache.indexForCodepoint(alloc, i, .regular, null)).?;
         try testing.expectEqual(Style.regular, idx.style);
         try testing.expectEqual(@as(Group.FontIndex.IndexInt, 0), idx.idx);
 
@@ -240,7 +248,7 @@ test {
 
         i = 32;
         while (i < 127) : (i += 1) {
-            const idx = (try cache.indexForCodepoint(alloc, .regular, i)).?;
+            const idx = (try cache.indexForCodepoint(alloc, i, .regular, null)).?;
             try testing.expectEqual(Style.regular, idx.style);
             try testing.expectEqual(@as(Group.FontIndex.IndexInt, 0), idx.idx);
 

src/font/Shaper.zig

@@ -179,14 +179,16 @@ pub const RunIterator = struct {
             // for unknown glyphs.
             const font_idx_opt = (try self.shaper.group.indexForCodepoint(
                 alloc,
-                style,
                 cell.char,
+                style,
+                null,
             )) orelse (try self.shaper.group.indexForCodepoint(
                 alloc,
-                style,
                 0xFFFD,
+                style,
+                null,
             )) orelse
-                try self.shaper.group.indexForCodepoint(alloc, style, ' ');
+                try self.shaper.group.indexForCodepoint(alloc, ' ', style, null);
             const font_idx = font_idx_opt.?;
             //log.warn("char={x} idx={}", .{ cell.char, font_idx });
             if (j == self.i) current_font = font_idx;
@@ -378,6 +380,40 @@ test "shape emoji width" {
     }
 }
 
+// test "shape variation selector VS15" {
+//     const testing = std.testing;
+//     const alloc = testing.allocator;
+//
+//     var testdata = try testShaper(alloc);
+//     defer testdata.deinit();
+//
+//     var buf: [32]u8 = undefined;
+//     var buf_idx: usize = 0;
+//     buf_idx += try std.unicode.utf8Encode(0x263A, buf[buf_idx..]); // White smiling face (text)
+//     buf_idx += try std.unicode.utf8Encode(0xFE0F, buf[buf_idx..]); // ZWJ to force color
+//
+//     // Make a screen with some data
+//     var screen = try terminal.Screen.init(alloc, 3, 10, 0);
+//     defer screen.deinit();
+//     try screen.testWriteString(buf[0..buf_idx]);
+//
+//     // Get our run iterator
+//     var shaper = testdata.shaper;
+//     var it = shaper.runIterator(screen.getRow(.{ .screen = 0 }));
+//     var count: usize = 0;
+//     while (try it.next(alloc)) |run| {
+//         count += 1;
+//         //try testing.expectEqual(@as(u32, 2), shaper.hb_buf.getLength());
+//
+//         const cells = try shaper.shape(run);
+//         try testing.expectEqual(@as(usize, 2), cells.len);
+//         log.warn("WHAT={}", .{cells[0]});
+//         log.warn("WHAT={}", .{cells[1]});
+//         try testing.expectEqual(@as(u8, 2), cells[0].width);
+//     }
+//     try testing.expectEqual(@as(usize, 1), count);
+// }
+
 const TestShaper = struct {
     alloc: Allocator,
     shaper: Shaper,

src/font/convert.zig

@@ -0,0 +1,73 @@
+//! Various conversions from Freetype formats to Atlas formats. These are
+//! currently implemented naively. There are definitely MUCH faster ways
+//! to do this (likely using SIMD), but I started simple.
+const std = @import("std");
+const freetype = @import("freetype");
+const Atlas = @import("../Atlas.zig");
+const assert = std.debug.assert;
+const Allocator = std.mem.Allocator;
+
+/// The mapping from freetype format to atlas format.
+pub const map = genMap();
+
+/// The map type.
+pub const Map = [freetype.c.FT_PIXEL_MODE_MAX]AtlasArray;
+
+/// Conversion function type. The returning bitmap buffer is guaranteed
+/// to be exactly `width * rows * depth` long for freeing it. The caller must
+/// free the bitmap buffer. The depth is the depth of the atlas format in the
+/// map.
+pub const Func = fn (Allocator, Bitmap) Allocator.Error!Bitmap;
+
+/// Alias for the freetype FT_Bitmap type to make it easier to type.
+pub const Bitmap = freetype.c.struct_FT_Bitmap_;
+
+const AtlasArray = std.EnumArray(Atlas.Format, ?Func);
+
+fn genMap() Map {
+    var result: Map = undefined;
+
+    // Initialize to no converter
+    var i: usize = 0;
+    while (i < freetype.c.FT_PIXEL_MODE_MAX) : (i += 1) {
+        result[i] = AtlasArray.initFill(null);
+    }
+
+    // Map our converters
+    result[freetype.c.FT_PIXEL_MODE_MONO].set(.rgba, monoToRGBA);
+
+    return result;
+}
+
+pub fn monoToRGBA(alloc: Allocator, bm: Bitmap) Allocator.Error!Bitmap {
+    // NOTE: This was never tested and may not work. I wrote it to
+    // solve another issue where this ended up not being needed.
+    // TODO: test this!
+
+    const depth = Atlas.Format.rgba.depth();
+    var buf = try alloc.alloc(u8, bm.width * bm.rows * depth);
+    errdefer alloc.free(buf);
+
+    var i: usize = 0;
+    while (i < bm.width * bm.rows) : (i += 1) {
+        var bit: u3 = 0;
+        while (bit <= 7) : (bit += 1) {
+            const mask = @as(u8, 1) << (7 - bit);
+            const bitval: u8 = if (bm.buffer[i] & mask > 0) 0xFF else 0;
+            const buf_i = (i * 8 * depth) + (bit * depth);
+            buf[buf_i] = 0xFF - bitval;
+            buf[buf_i + 1] = 0xFF - bitval;
+            buf[buf_i + 2] = 0xFF - bitval;
+            buf[buf_i + 3] = bitval;
+        }
+    }
+
+    var copy = bm;
+    copy.buffer = buf.ptr;
+    copy.pixel_mode = freetype.c.FT_PIXEL_MODE_BGRA;
+    return copy;
+}
+
+test {
+    _ = map;
+}

src/font/main.zig

@@ -8,13 +8,19 @@ pub const Library = @import("Library.zig");
 pub const Shaper = @import("Shaper.zig");
 
 /// The styles that a family can take.
-pub const Style = enum(u2) {
+pub const Style = enum(u3) {
     regular = 0,
     bold = 1,
     italic = 2,
     bold_italic = 3,
 };
 
+/// The presentation for a an emoji.
+pub const Presentation = enum(u1) {
+    text = 0, // U+FE0E
+    emoji = 1, // U+FEOF
+};
+
 /// Font metrics useful for things such as grid calculation.
 pub const Metrics = struct {
     /// The width and height of a monospace cell.

src/font/test.zig

@@ -1,3 +1,4 @@
 pub const fontRegular = @embedFile("res/Inconsolata-Regular.ttf");
 pub const fontBold = @embedFile("res/Inconsolata-Bold.ttf");
 pub const fontEmoji = @embedFile("res/NotoColorEmoji.ttf");
+pub const fontEmojiText = @embedFile("res/NotoEmoji-Regular.ttf");