Merge pull request #11 from mitchellh/graphemes

Graphemes and ZWJs

src/Window.zig

@@ -1499,7 +1499,7 @@ pub fn eraseChars(self: *Window, count: usize) !void {
 }
 
 pub fn insertLines(self: *Window, count: usize) !void {
-    self.terminal.insertLines(count);
+    try self.terminal.insertLines(count);
 }
 
 pub fn insertBlanks(self: *Window, count: usize) !void {
@@ -1507,7 +1507,7 @@ pub fn insertBlanks(self: *Window, count: usize) !void {
 }
 
 pub fn deleteLines(self: *Window, count: usize) !void {
-    self.terminal.deleteLines(count);
+    try self.terminal.deleteLines(count);
 }
 
 pub fn reverseIndex(self: *Window) !void {
@@ -1663,7 +1663,7 @@ pub fn setCursorStyle(
 }
 
 pub fn decaln(self: *Window) !void {
-    self.terminal.decaln();
+    try self.terminal.decaln();
 }
 
 pub fn tabClear(self: *Window, cmd: terminal.TabClear) !void {
@@ -1687,11 +1687,11 @@ pub fn enquiry(self: *Window) !void {
 }
 
 pub fn scrollDown(self: *Window, count: usize) !void {
-    self.terminal.scrollDown(count);
+    try self.terminal.scrollDown(count);
 }
 
 pub fn scrollUp(self: *Window, count: usize) !void {
-    self.terminal.scrollUp(count);
+    try self.terminal.scrollUp(count);
 }
 
 pub fn setActiveStatusDisplay(

src/font/Shaper.zig

@@ -107,6 +107,15 @@ pub const RunIterator = struct {
 
             // Continue with our run
             self.shaper.hb_buf.add(cell.char, @intCast(u32, j));
+
+            // If this cell is part of a grapheme cluster, add all the grapheme
+            // data points.
+            if (cell.attrs.grapheme) {
+                var it = self.row.codepointIterator(j);
+                while (it.next()) |cp| {
+                    self.shaper.hb_buf.add(cp, @intCast(u32, j));
+                }
+            }
         }
 
         // Finalize our buffer

src/terminal/Screen.zig

@@ -97,7 +97,7 @@ const StorageCell = union {
     comptime {
         // We only check this during ReleaseFast because safety checks
         // have to be disabled to get this size.
-        if (builtin.mode == .ReleaseFast) {
+        if (!std.debug.runtime_safety) {
             // We want to be at most the size of a cell always. We have WAY
             // more cells than other fields, so we don't want to pay the cost
             // of padding due to other fields.
@@ -113,12 +113,23 @@ const StorageCell = union {
 /// The row header is at the start of every row within the storage buffer.
 /// It can store row-specific data.
 pub const RowHeader = struct {
-    /// Used internally to track if this row has been initialized.
+    const Id = u32;
-    init: bool = false,
 
-    /// If true, this row is soft-wrapped. The first cell of the next
+    /// The ID of this row, used to uniquely identify this row. The cells
-    /// row is a continuous of this row.
+    /// are also ID'd by id + cell index (0-indexed). This will wrap around
-    wrap: bool = false,
+    /// when it reaches the maximum value for the type. For caching purposes,
+    /// when wrapping happens, all rows in the screen will be marked dirty.
+    id: Id = 0,
+
+    // Packed flags
+    flags: packed struct {
+        /// If true, this row is soft-wrapped. The first cell of the next
+        /// row is a continuous of this row.
+        wrap: bool = false,
+
+        /// True if any cell in this row has a grapheme associated with it.
+        grapheme: bool = false,
+    } = .{},
 };
 
 /// Cell is a single cell within the screen.
@@ -136,8 +147,8 @@ pub const Cell = struct {
 
     /// Foreground and background color. attrs.has_{bg/fg} must be checked
     /// to see if these are useful values.
-    fg: color.RGB = undefined,
+    fg: color.RGB = .{},
-    bg: color.RGB = undefined,
+    bg: color.RGB = .{},
 
     /// On/off attributes that can be set
     attrs: packed struct {
@@ -157,6 +168,12 @@ pub const Cell = struct {
         /// wide character (tail) or following (head).
         wide_spacer_tail: bool = false,
         wide_spacer_head: bool = false,
+
+        /// True if this cell has additional codepoints to form a complete
+        /// grapheme cluster. If this is true, then the row grapheme flag must
+        /// also be true. The grapheme code points can be looked up in the
+        /// screen grapheme map.
+        grapheme: bool = false,
     } = .{},
 
     /// True if the cell should be skipped for drawing
@@ -179,15 +196,24 @@ pub const Cell = struct {
 
 /// A row is a single row in the screen.
 pub const Row = struct {
+    /// The screen this row is part of.
+    screen: *Screen,
+
     /// Raw internal storage, do NOT write to this, use only the
     /// helpers. Writing directly to this can easily mess up state
     /// causing future crashes or misrendering.
     storage: []StorageCell,
 
+    /// Returns the ID for this row. You can turn this into a cell ID
+    /// by adding the cell offset plus 1 (so it is 1-indexed).
+    pub fn getId(self: Row) RowHeader.Id {
+        return self.storage[0].header.id;
+    }
+
     /// Set that this row is soft-wrapped. This doesn't change the contents
     /// of this row so the row won't be marked dirty.
     pub fn setWrapped(self: Row, v: bool) void {
-        self.storage[0].header.wrap = v;
+        self.storage[0].header.flags.wrap = v;
     }
 
     /// Retrieve the header for this row.
@@ -209,18 +235,36 @@ pub const Row = struct {
 
     /// Fill the entire row with a copy of a single cell.
     pub fn fill(self: Row, cell: Cell) void {
-        std.mem.set(StorageCell, self.storage[1..], .{ .cell = cell });
+        self.fillSlice(cell, 0, self.storage.len - 1);
     }
 
     /// Fill a slice of a row.
     pub fn fillSlice(self: Row, cell: Cell, start: usize, len: usize) void {
         assert(len <= self.storage.len - 1);
-        std.mem.set(StorageCell, self.storage[start + 1 .. len + 1], .{ .cell = cell });
+        assert(!cell.attrs.grapheme); // you can't fill with graphemes
+
+        // If our row has no graphemes, then this is a fast copy
+        if (!self.storage[0].header.flags.grapheme) {
+            std.mem.set(StorageCell, self.storage[start + 1 .. len + 1], .{ .cell = cell });
+            return;
+        }
+
+        // We have graphemes, so we have to clear those first.
+        for (self.storage[start + 1 .. len + 1]) |*storage_cell, x| {
+            if (storage_cell.cell.attrs.grapheme) self.clearGraphemes(x);
+            storage_cell.* = .{ .cell = cell };
+        }
+
+        // We only reset the grapheme flag if we fill the whole row, for now.
+        // We can improve performance by more correctly setting this but I'm
+        // going to defer that until we can measure.
+        if (start == 0 and len == self.storage.len - 1) {
+            self.storage[0].header.flags.grapheme = false;
+        }
     }
 
     /// Get a single immutable cell.
     pub fn getCell(self: Row, x: usize) Cell {
-        assert(self.header().init);
         assert(x < self.storage.len - 1);
         return self.storage[x + 1].cell;
     }
@@ -230,31 +274,96 @@ pub const Row = struct {
     /// next call to re-render this cell. Any change detection to avoid
     /// this should be done prior.
     pub fn getCellPtr(self: Row, x: usize) *Cell {
-        assert(self.header().init);
         assert(x < self.storage.len - 1);
         return &self.storage[x + 1].cell;
     }
 
+    /// Attach a grapheme codepoint to the given cell.
+    pub fn attachGrapheme(self: Row, x: usize, cp: u21) !void {
+        const cell = &self.storage[x + 1].cell;
+        const key = self.getId() + x + 1;
+        const gop = try self.screen.graphemes.getOrPut(self.screen.alloc, key);
+        errdefer if (!gop.found_existing) {
+            _ = self.screen.graphemes.remove(key);
+        };
+
+        // Our row now has a grapheme
+        self.storage[0].header.flags.grapheme = true;
+
+        // If we weren't previously a grapheme and we found an existing value
+        // it means that it is old grapheme data. Just delete that.
+        if (!cell.attrs.grapheme and gop.found_existing) {
+            cell.attrs.grapheme = true;
+            gop.value_ptr.deinit(self.screen.alloc);
+            gop.value_ptr.* = .{ .one = cp };
+            return;
+        }
+
+        // If we didn't have a previous value, attach the single codepoint.
+        if (!gop.found_existing) {
+            cell.attrs.grapheme = true;
+            gop.value_ptr.* = .{ .one = cp };
+            return;
+        }
+
+        // We have an existing value, promote
+        assert(cell.attrs.grapheme);
+        try gop.value_ptr.append(self.screen.alloc, cp);
+    }
+
+    /// Removes all graphemes associated with a cell.
+    pub fn clearGraphemes(self: Row, x: usize) void {
+        const cell = &self.storage[x + 1].cell;
+        const key = self.getId() + x + 1;
+        cell.attrs.grapheme = false;
+        _ = self.screen.graphemes.remove(key);
+    }
+
     /// Copy the row src into this row. The row can be from another screen.
-    pub fn copyRow(self: Row, src: Row) void {
+    pub fn copyRow(self: Row, src: Row) !void {
-        assert(self.header().init);
+        // If we have graphemes, clear first to unset them.
+        if (self.storage[0].header.flags.grapheme) self.clear(.{});
+
+        // If the source has no graphemes (likely) then this is fast.
         const end = @minimum(src.storage.len, self.storage.len);
-        std.mem.copy(StorageCell, self.storage[1..], src.storage[1..end]);
+        if (!src.storage[0].header.flags.grapheme) {
+            std.mem.copy(StorageCell, self.storage[1..], src.storage[1..end]);
+            return;
+        }
+
+        // Source has graphemes, this is slow.
+        for (src.storage[1..end]) |storage, x| {
+            self.storage[x + 1] = .{ .cell = storage.cell };
+
+            // Copy grapheme data if it exists
+            if (storage.cell.attrs.grapheme) {
+                const src_key = src.getId() + x + 1;
+                const src_data = src.screen.graphemes.get(src_key) orelse continue;
+
+                const dst_key = self.getId() + x + 1;
+                const dst_gop = try self.screen.graphemes.getOrPut(self.screen.alloc, dst_key);
+                dst_gop.value_ptr.* = try src_data.copy(self.screen.alloc);
+
+                self.storage[0].header.flags.grapheme = true;
+            }
+        }
     }
 
     /// Read-only iterator for the cells in the row.
     pub fn cellIterator(self: Row) CellIterator {
-        assert(self.header().init);
         return .{ .row = self };
     }
 
-    /// If this row isn't initialized, this sets all our cells to the
+    /// Read-only iterator for the grapheme codepoints in a cell. This only
-    /// proper union tag so that it is properly zeroed.
+    /// iterates over the EXTRA GRAPHEME codepoints and not the primary
-    fn initIfNeeded(self: Row) void {
+    /// codepoint in cell.char.
-        if (!self.storage[0].header.init) {
+    pub fn codepointIterator(self: Row, x: usize) CodepointIterator {
-            self.fill(.{});
+        const cell = &self.storage[x + 1].cell;
-            self.storage[0].header.init = true;
+        assert(cell.attrs.grapheme);
-        }
+
+        const key = self.getId() + x + 1;
+        const data = self.screen.graphemes.get(key).?;
+        return .{ .data = data };
     }
 };
 
@@ -287,6 +396,47 @@ pub const CellIterator = struct {
     }
 };
 
+/// Used to iterate through the codepoints of a cell. This only iterates
+/// over the extra grapheme codepoints and not the primary codepoint.
+pub const CodepointIterator = struct {
+    data: GraphemeData,
+    i: usize = 0,
+
+    pub fn next(self: *CodepointIterator) ?u21 {
+        switch (self.data) {
+            .one => |v| {
+                if (self.i >= 1) return null;
+                self.i += 1;
+                return v;
+            },
+
+            .two => |v| {
+                if (self.i >= v.len) return null;
+                defer self.i += 1;
+                return v[self.i];
+            },
+
+            .three => |v| {
+                if (self.i >= v.len) return null;
+                defer self.i += 1;
+                return v[self.i];
+            },
+
+            .four => |v| {
+                if (self.i >= v.len) return null;
+                defer self.i += 1;
+                return v[self.i];
+            },
+
+            .many => |v| {
+                if (self.i >= v.len) return null;
+                defer self.i += 1;
+                return v[self.i];
+            },
+        }
+    }
+};
+
 /// RowIndex represents a row within the screen. There are various meanings
 /// of a row index and this union represents the available types. For example,
 /// when talking about row "0" you may want the first row in the viewport,
@@ -389,16 +539,112 @@ pub const RowIndexTag = enum {
     }
 };
 
+/// Stores the extra unicode codepoints that form a complete grapheme
+/// cluster alongside a cell. We store this separately from a Cell because
+/// grapheme clusters are relatively rare (depending on the language) and
+/// we don't want to pay for the full cost all the time.
+pub const GraphemeData = union(enum) {
+    // The named counts allow us to avoid allocators. We do this because
+    // []u21 is sizeof([4]u21) anyways so if we can store avoid small allocations
+    // we prefer it. Grapheme clusters are almost always <= 4 codepoints.
+
+    one: u21,
+    two: [2]u21,
+    three: [3]u21,
+    four: [4]u21,
+    many: []u21,
+
+    pub fn deinit(self: GraphemeData, alloc: Allocator) void {
+        switch (self) {
+            .many => |v| alloc.free(v),
+            else => {},
+        }
+    }
+
+    /// Append the codepoint cp to the grapheme data.
+    pub fn append(self: *GraphemeData, alloc: Allocator, cp: u21) !void {
+        switch (self.*) {
+            .one => |v| self.* = .{ .two = .{ v, cp } },
+            .two => |v| self.* = .{ .three = .{ v[0], v[1], cp } },
+            .three => |v| self.* = .{ .four = .{ v[0], v[1], v[2], cp } },
+            .four => |v| {
+                const many = try alloc.alloc(u21, 5);
+                std.mem.copy(u21, many, &v);
+                many[4] = cp;
+                self.* = .{ .many = many };
+            },
+
+            .many => |v| {
+                // Note: this is super inefficient, we should use an arraylist
+                // or something so we have extra capacity.
+                const many = try alloc.realloc(v, v.len + 1);
+                many[v.len] = cp;
+                self.* = .{ .many = many };
+            },
+        }
+    }
+
+    pub fn copy(self: GraphemeData, alloc: Allocator) !GraphemeData {
+        // If we're not many we're not allocated so just copy on stack.
+        if (self != .many) return self;
+
+        // Heap allocated
+        return GraphemeData{ .many = try alloc.dupe(u21, self.many) };
+    }
+
+    test {
+        log.warn("Grapheme={}", .{@sizeOf(GraphemeData)});
+    }
+
+    test "append" {
+        const testing = std.testing;
+        const alloc = testing.allocator;
+
+        var data: GraphemeData = .{ .one = 1 };
+        defer data.deinit(alloc);
+
+        try data.append(alloc, 2);
+        try testing.expectEqual(GraphemeData{ .two = .{ 1, 2 } }, data);
+        try data.append(alloc, 3);
+        try testing.expectEqual(GraphemeData{ .three = .{ 1, 2, 3 } }, data);
+        try data.append(alloc, 4);
+        try testing.expectEqual(GraphemeData{ .four = .{ 1, 2, 3, 4 } }, data);
+        try data.append(alloc, 5);
+        try testing.expect(data == .many);
+        try testing.expectEqualSlices(u21, &[_]u21{ 1, 2, 3, 4, 5 }, data.many);
+        try data.append(alloc, 6);
+        try testing.expect(data == .many);
+        try testing.expectEqualSlices(u21, &[_]u21{ 1, 2, 3, 4, 5, 6 }, data.many);
+    }
+
+    comptime {
+        // We want to keep this at most the size of the tag + []u21 so that
+        // at most we're paying for the cost of a slice.
+        //assert(@sizeOf(GraphemeData) == 24);
+    }
+};
+
 // Initialize to header and not a cell so that we can check header.init
 // to know if the remainder of the row has been initialized or not.
 const StorageBuf = CircBuf(StorageCell, .{ .header = .{} });
 
+/// Stores a mapping of cell ID (row ID + cell offset + 1) to
+/// graphemes associated with a cell. To know if a cell has graphemes,
+/// check the "grapheme" flag of a cell.
+const GraphemeMap = std.AutoHashMapUnmanaged(usize, GraphemeData);
+
 /// The allocator used for all the storage operations
 alloc: Allocator,
 
 /// The full set of storage.
 storage: StorageBuf,
 
+/// Graphemes associated with our current screen.
+graphemes: GraphemeMap = .{},
+
+/// The next ID to assign to a row. The value of this is NOT assigned.
+next_row_id: RowHeader.Id = 1,
+
 /// The number of rows and columns in the visible space.
 rows: usize,
 cols: usize,
@@ -448,6 +694,10 @@ pub fn init(
 
 pub fn deinit(self: *Screen) void {
     self.storage.deinit(self.alloc);
+
+    var grapheme_it = self.graphemes.valueIterator();
+    while (grapheme_it.next()) |data| if (data.* == .many) self.alloc.free(data.many);
+    self.graphemes.deinit(self.alloc);
 }
 
 /// Returns true if the viewport is scrolled to the bottom of the screen.
@@ -495,18 +745,29 @@ pub fn getRow(self: *Screen, index: RowIndex) Row {
     const slices = self.storage.getPtrSlice(offset, self.cols + 1);
     assert(slices[0].len == self.cols + 1 and slices[1].len == 0);
 
-    const row: Row = .{ .storage = slices[0] };
+    const row: Row = .{ .screen = self, .storage = slices[0] };
-    row.initIfNeeded();
+    if (row.storage[0].header.id == 0) {
+        const Id = @TypeOf(self.next_row_id);
+        const id = self.next_row_id;
+        self.next_row_id +%= @intCast(Id, self.cols);
+
+        // Store the header
+        row.storage[0].header.id = id;
+
+        // We only need to fill with runtime safety because unions are
+        // tag-checked. Otherwise, the default value of zero will be valid.
+        if (std.debug.runtime_safety) row.fill(.{});
+    }
     return row;
 }
 
 /// Copy the row at src to dst.
-pub fn copyRow(self: *Screen, dst: RowIndex, src: RowIndex) void {
+pub fn copyRow(self: *Screen, dst: RowIndex, src: RowIndex) !void {
     // One day we can make this more efficient but for now
     // we do the easy thing.
     const dst_row = self.getRow(dst);
     const src_row = self.getRow(src);
-    dst_row.copyRow(src_row);
+    try dst_row.copyRow(src_row);
 }
 
 /// Returns the offset into the storage buffer that the given row can
@@ -656,6 +917,19 @@ fn scrollDelta(self: *Screen, delta: isize, grow: bool) !void {
         // If we can't fit our rows into our capacity, we delete some scrollback.
         const rows_deleted = if (rows_final > self.rowsCapacity()) deleted: {
             const rows_to_delete = rows_final - self.rowsCapacity();
+
+            // Fast-path: we have no graphemes.
+            // Slow-path: we have graphemes, we have to check each row
+            // we're going to delete to see if they contain graphemes and
+            // clear the ones that do so we clear memory properly.
+            if (self.graphemes.count() > 0) {
+                var y: usize = 0;
+                while (y < rows_to_delete) : (y += 1) {
+                    const row = self.getRow(.{ .active = y });
+                    if (row.storage[0].header.flags.grapheme) row.clear(.{});
+                }
+            }
+
             self.viewport -= rows_to_delete;
             self.storage.deleteOldest(rows_to_delete * (self.cols + 1));
             break :deleted rows_to_delete;
@@ -733,7 +1007,7 @@ pub fn selectionString(self: *Screen, alloc: Allocator, sel: Selection) ![:0]con
             // the first row.
             var skip: usize = if (row_count == 0) slices.top_offset else 0;
 
-            const row: Row = .{ .storage = slice[start_idx..end_idx] };
+            const row: Row = .{ .screen = self, .storage = slice[start_idx..end_idx] };
             var it = row.cellIterator();
             while (it.next()) |cell| {
                 if (skip > 0) {
@@ -750,7 +1024,7 @@ pub fn selectionString(self: *Screen, alloc: Allocator, sel: Selection) ![:0]con
             }
 
             // If this row is not soft-wrapped, add a newline
-            if (!row.header().wrap) {
+            if (!row.header().flags.wrap) {
                 buf[buf_i] = '\n';
                 buf_i += 1;
             }
@@ -883,7 +1157,7 @@ pub fn resizeWithoutReflow(self: *Screen, rows: usize, cols: usize) !void {
 
         // Get this row
         const new_row = self.getRow(.{ .active = y });
-        new_row.copyRow(old_row);
+        try new_row.copyRow(old_row);
 
         // Next row
         y += 1;
@@ -965,7 +1239,7 @@ pub fn resize(self: *Screen, rows: usize, cols: usize) !void {
 
             // Get this row
             var new_row = self.getRow(.{ .active = y });
-            new_row.copyRow(old_row);
+            try new_row.copyRow(old_row);
 
             // We need to check if our cursor was on this line. If so,
             // we set the new cursor.
@@ -975,7 +1249,7 @@ pub fn resize(self: *Screen, rows: usize, cols: usize) !void {
             }
 
             // If no reflow, just keep going
-            if (!old_row.header().wrap) {
+            if (!old_row.header().flags.wrap) {
                 y += 1;
                 continue;
             }
@@ -1029,7 +1303,7 @@ pub fn resize(self: *Screen, rows: usize, cols: usize) !void {
                     // We copied the full amount left in this wrapped row.
                     if (copy_len == wrapped_cells_rem) {
                         // If this row isn't also wrapped, we're done!
-                        if (!wrapped_row.header().wrap) {
+                        if (!wrapped_row.header().flags.wrap) {
                             // If we were able to copy the entire row then
                             // we shortened the screen by one. We need to reflect
                             // this in our viewport.
@@ -1168,7 +1442,7 @@ pub fn resize(self: *Screen, rows: usize, cols: usize) !void {
 
             // If we aren't wrapping, then move to the next row
             if (trimmed_row.len == 0 or
-                !old_row.header().wrap)
+                !old_row.header().flags.wrap)
             {
                 y += 1;
                 x = 0;
@@ -1195,8 +1469,13 @@ pub fn resize(self: *Screen, rows: usize, cols: usize) !void {
 /// each row. If a line is longer than the available columns, soft-wrapping
 /// will occur. This will automatically handle basic wide chars.
 pub fn testWriteString(self: *Screen, text: []const u8) !void {
-    var y: usize = 0;
+    var y: usize = self.cursor.y;
-    var x: usize = 0;
+    var x: usize = self.cursor.x;
+
+    var grapheme: struct {
+        x: usize = 0,
+        cell: ?*Cell = null,
+    } = .{};
 
     const view = std.unicode.Utf8View.init(text) catch unreachable;
     var iter = view.iterator();
@@ -1205,6 +1484,7 @@ pub fn testWriteString(self: *Screen, text: []const u8) !void {
         if (c == '\n') {
             y += 1;
             x = 0;
+            grapheme = .{};
             continue;
         }
 
@@ -1217,6 +1497,33 @@ pub fn testWriteString(self: *Screen, text: []const u8) !void {
         // Get our row
         var row = self.getRow(.{ .active = y });
 
+        // If we have a previous cell, we check if we're part of a grapheme.
+        if (grapheme.cell) |prev_cell| {
+            const grapheme_break = brk: {
+                var state: i32 = 0;
+                var cp1 = @intCast(u21, prev_cell.char);
+                if (prev_cell.attrs.grapheme) {
+                    var it = row.codepointIterator(grapheme.x);
+                    while (it.next()) |cp2| {
+                        assert(!utf8proc.graphemeBreakStateful(
+                            cp1,
+                            cp2,
+                            &state,
+                        ));
+
+                        cp1 = cp2;
+                    }
+                }
+
+                break :brk utf8proc.graphemeBreakStateful(cp1, c, &state);
+            };
+
+            if (!grapheme_break) {
+                try row.attachGrapheme(grapheme.x, c);
+                continue;
+            }
+        }
+
         // If we're writing past the end, we need to soft wrap.
         if (x == self.cols) {
             row.setWrapped(true);
@@ -1236,6 +1543,9 @@ pub fn testWriteString(self: *Screen, text: []const u8) !void {
             1 => {
                 const cell = row.getCellPtr(x);
                 cell.char = @intCast(u32, c);
+
+                grapheme.x = x;
+                grapheme.cell = cell;
             },
 
             2 => {
@@ -1259,6 +1569,9 @@ pub fn testWriteString(self: *Screen, text: []const u8) !void {
                     const cell = row.getCellPtr(x);
                     cell.char = @intCast(u32, c);
                     cell.attrs.wide = true;
+
+                    grapheme.x = x;
+                    grapheme.cell = cell;
                 }
 
                 {
@@ -1274,6 +1587,10 @@ pub fn testWriteString(self: *Screen, text: []const u8) !void {
 
         x += 1;
     }
+
+    // So the cursor doesn't go off screen
+    self.cursor.x = @minimum(x, self.cols - 1);
+    self.cursor.y = y;
 }
 
 /// Turns the screen into a string. Different regions of the screen can
@@ -1309,6 +1626,93 @@ pub fn testString(self: *Screen, alloc: Allocator, tag: RowIndexTag) ![]const u8
     return try alloc.realloc(buf, str.len);
 }
 
+test "Row: clear with graphemes" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var s = try init(alloc, 5, 5, 0);
+    defer s.deinit();
+
+    const row = s.getRow(.{ .active = 0 });
+    try testing.expect(row.getId() > 0);
+    try testing.expectEqual(@as(usize, 5), row.lenCells());
+    try testing.expect(!row.header().flags.grapheme);
+
+    // Lets add a cell with a grapheme
+    {
+        const cell = row.getCellPtr(2);
+        cell.*.char = 'A';
+        try row.attachGrapheme(2, 'B');
+        try testing.expect(cell.attrs.grapheme);
+        try testing.expect(row.header().flags.grapheme);
+        try testing.expect(s.graphemes.count() == 1);
+    }
+
+    // Clear the row
+    row.clear(.{});
+    try testing.expect(!row.header().flags.grapheme);
+    try testing.expect(s.graphemes.count() == 0);
+}
+
+test "Row: copy row with graphemes in destination" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var s = try init(alloc, 5, 5, 0);
+    defer s.deinit();
+
+    // Source row does NOT have graphemes
+    const row_src = s.getRow(.{ .active = 0 });
+    {
+        const cell = row_src.getCellPtr(2);
+        cell.*.char = 'A';
+    }
+
+    // Destination has graphemes
+    const row = s.getRow(.{ .active = 1 });
+    {
+        const cell = row.getCellPtr(1);
+        cell.*.char = 'B';
+        try row.attachGrapheme(1, 'C');
+        try testing.expect(cell.attrs.grapheme);
+        try testing.expect(row.header().flags.grapheme);
+        try testing.expect(s.graphemes.count() == 1);
+    }
+
+    // Copy
+    try row.copyRow(row_src);
+    try testing.expect(!row.header().flags.grapheme);
+    try testing.expect(s.graphemes.count() == 0);
+}
+
+test "Row: copy row with graphemes in source" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var s = try init(alloc, 5, 5, 0);
+    defer s.deinit();
+
+    // Source row does NOT have graphemes
+    const row_src = s.getRow(.{ .active = 0 });
+    {
+        const cell = row_src.getCellPtr(2);
+        cell.*.char = 'A';
+        try row_src.attachGrapheme(2, 'B');
+        try testing.expect(cell.attrs.grapheme);
+        try testing.expect(row_src.header().flags.grapheme);
+        try testing.expect(s.graphemes.count() == 1);
+    }
+
+    // Destination has no graphemes
+    const row = s.getRow(.{ .active = 1 });
+    try row.copyRow(row_src);
+    try testing.expect(row.header().flags.grapheme);
+    try testing.expect(s.graphemes.count() == 2);
+
+    row_src.clear(.{});
+    try testing.expect(s.graphemes.count() == 1);
+}
+
 test "Screen" {
     const testing = std.testing;
     const alloc = testing.allocator;
@@ -1350,6 +1754,25 @@ test "Screen" {
     }
 }
 
+test "Screen: write graphemes" {
+    const testing = std.testing;
+    const alloc = testing.allocator;
+
+    var s = try init(alloc, 5, 5, 0);
+    defer s.deinit();
+
+    // Sanity check that our test helpers work
+    var buf: [32]u8 = undefined;
+    var buf_idx: usize = 0;
+    buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain
+    buf_idx += try std.unicode.utf8Encode(0x1F44D, buf[buf_idx..]); // Thumbs up plain
+    buf_idx += try std.unicode.utf8Encode(0x1F3FD, buf[buf_idx..]); // Medium skin tone
+
+    try s.testWriteString(buf[0..buf_idx]);
+    try testing.expect(s.rowsWritten() == 1);
+    try testing.expectEqual(@as(usize, 4), s.cursor.x);
+}
+
 test "Screen: scrolling" {
     const testing = std.testing;
     const alloc = testing.allocator;
@@ -1609,7 +2032,7 @@ test "Screen: row copy" {
 
     // Copy
     try s.scroll(.{ .delta = 1 });
-    s.copyRow(.{ .active = 2 }, .{ .active = 0 });
+    try s.copyRow(.{ .active = 2 }, .{ .active = 0 });
 
     // Test our contents
     var contents = try s.testString(alloc, .viewport);
@@ -2217,6 +2640,8 @@ test "Screen: resize less rows no scrollback" {
     defer s.deinit();
     const str = "1ABCD\n2EFGH\n3IJKL";
     try s.testWriteString(str);
+    s.cursor.x = 0;
+    s.cursor.y = 0;
     const cursor = s.cursor;
     try s.resize(1, 5);
 
@@ -2334,6 +2759,8 @@ test "Screen: resize less cols no reflow" {
     defer s.deinit();
     const str = "1AB\n2EF\n3IJ";
     try s.testWriteString(str);
+    s.cursor.x = 0;
+    s.cursor.y = 0;
     const cursor = s.cursor;
     try s.resize(3, 3);
 

src/terminal/Terminal.zig

@@ -449,14 +449,83 @@ pub fn print(self: *Terminal, c: u21) !void {
     // If we're not on the main display, do nothing for now
     if (self.status_display != .main) return;
 
+    // Get the previous cell so we can detect grapheme clusters. We only
+    // do this if c is outside of Latin-1 because characters in the Latin-1
+    // range cannot possibly be grapheme joiners. This helps keep non-graphemes
+    // extremely fast and we take this much slower path for graphemes. No hate
+    // on graphemes, I'd love to make them much faster, but I wanted to focus
+    // on correctness first.
+    //
+    // NOTE: This is disabled because no shells handle this correctly. We'll
+    // need to work with shells and other emulators to probably figure out
+    // a way to support this. In the mean time, I'm going to keep all the
+    // grapheme detection and keep it up to date so we're ready to go.
+    if (false and c > 255 and self.screen.cursor.x > 0) {
+        // TODO: test this!
+
+        const row = self.screen.getRow(.{ .active = self.screen.cursor.y });
+        const Prev = struct { cell: *Screen.Cell, x: usize };
+        const prev: Prev = prev: {
+            const x = self.screen.cursor.x - 1;
+            const immediate = row.getCellPtr(x);
+            if (!immediate.attrs.wide_spacer_tail) break :prev .{
+                .cell = immediate,
+                .x = x,
+            };
+
+            break :prev .{
+                .cell = row.getCellPtr(x - 1),
+                .x = x - 1,
+            };
+        };
+
+        const grapheme_break = brk: {
+            var state: i32 = 0;
+            var cp1 = @intCast(u21, prev.cell.char);
+            if (prev.cell.attrs.grapheme) {
+                var it = row.codepointIterator(prev.x);
+                while (it.next()) |cp2| {
+                    assert(!utf8proc.graphemeBreakStateful(
+                        cp1,
+                        cp2,
+                        &state,
+                    ));
+
+                    cp1 = cp2;
+                }
+            }
+
+            break :brk utf8proc.graphemeBreakStateful(cp1, c, &state);
+        };
+
+        // If we can NOT break, this means that "c" is part of a grapheme
+        // with the previous char.
+        if (!grapheme_break) {
+            log.debug("c={x} grapheme attach to x={}", .{ c, prev.x });
+            try row.attachGrapheme(prev.x, c);
+            return;
+        }
+    }
+
     // Determine the width of this character so we can handle
     // non-single-width characters properly.
     const width = utf8proc.charwidth(c);
     assert(width <= 2);
 
-    // For now, we ignore zero-width characters. When we support ligatures,
+    // Attach zero-width characters to our cell as grapheme data.
-    // this will have to change.
+    if (width == 0) {
-    if (width == 0) return;
+        // Find our previous cell
+        const row = self.screen.getRow(.{ .active = self.screen.cursor.y });
+        const prev: usize = prev: {
+            const x = self.screen.cursor.x - 1;
+            const immediate = row.getCellPtr(x);
+            if (!immediate.attrs.wide_spacer_tail) break :prev x;
+            break :prev x - 1;
+        };
+
+        try row.attachGrapheme(prev, c);
+        return;
+    }
 
     // If we're soft-wrapping, then handle that first.
     if (self.screen.cursor.pending_wrap and self.modes.autowrap)
@@ -554,6 +623,9 @@ fn printCell(self: *Terminal, unmapped_c: u21) *Screen.Cell {
         }
     }
 
+    // If the prior value had graphemes, clear those
+    if (cell.attrs.grapheme) row.clearGraphemes(self.screen.cursor.x);
+
     // Write
     cell.* = self.screen.cursor.pen;
     cell.char = @intCast(u32, c);
@@ -586,7 +658,7 @@ fn clearWideSpacerHead(self: *Terminal) void {
 /// Resets all margins and fills the whole screen with the character 'E'
 ///
 /// Sets the cursor to the top left corner.
-pub fn decaln(self: *Terminal) void {
+pub fn decaln(self: *Terminal) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
@@ -600,7 +672,7 @@ pub fn decaln(self: *Terminal) void {
 
     var row: usize = 1;
     while (row < self.rows) : (row += 1) {
-        self.screen.getRow(.{ .active = row }).copyRow(filled);
+        try self.screen.getRow(.{ .active = row }).copyRow(filled);
     }
 }
 
@@ -643,7 +715,7 @@ pub fn index(self: *Terminal) !void {
             try self.screen.scroll(.{ .delta = 1 });
         } else {
             // TODO: test
-            self.scrollUp(1);
+            try self.scrollUp(1);
         }
 
         return;
@@ -672,7 +744,7 @@ pub fn reverseIndex(self: *Terminal) !void {
     // TODO: scrolling region
 
     if (self.screen.cursor.y == 0) {
-        self.scrollDown(1);
+        try self.scrollDown(1);
     } else {
         self.screen.cursor.y -|= 1;
     }
@@ -1074,7 +1146,7 @@ pub fn insertBlanks(self: *Terminal, count: usize) void {
 /// All cleared space is colored according to the current SGR state.
 ///
 /// Moves the cursor to the left margin.
-pub fn insertLines(self: *Terminal, count: usize) void {
+pub fn insertLines(self: *Terminal, count: usize) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
@@ -1095,7 +1167,7 @@ pub fn insertLines(self: *Terminal, count: usize) void {
 
     // Ensure we have the lines populated to the end
     while (y > top) : (y -= 1) {
-        self.screen.copyRow(.{ .active = y }, .{ .active = y - adjusted_count });
+        try self.screen.copyRow(.{ .active = y }, .{ .active = y - adjusted_count });
     }
 
     // Insert count blank lines
@@ -1122,7 +1194,7 @@ pub fn insertLines(self: *Terminal, count: usize) void {
 /// cleared space is colored according to the current SGR state.
 ///
 /// Moves the cursor to the left margin.
-pub fn deleteLines(self: *Terminal, count: usize) void {
+pub fn deleteLines(self: *Terminal, count: usize) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
@@ -1140,7 +1212,7 @@ pub fn deleteLines(self: *Terminal, count: usize) void {
     // Scroll up the count amount.
     var y: usize = self.screen.cursor.y;
     while (y <= self.scrolling_region.bottom - adjusted_count) : (y += 1) {
-        self.screen.copyRow(.{ .active = y }, .{ .active = y + adjusted_count });
+        try self.screen.copyRow(.{ .active = y }, .{ .active = y + adjusted_count });
     }
 
     while (y <= self.scrolling_region.bottom) : (y += 1) {
@@ -1151,7 +1223,7 @@ pub fn deleteLines(self: *Terminal, count: usize) void {
 
 /// Scroll the text down by one row.
 /// TODO: test
-pub fn scrollDown(self: *Terminal, count: usize) void {
+pub fn scrollDown(self: *Terminal, count: usize) !void {
     const tracy = trace(@src());
     defer tracy.end();
 
@@ -1161,7 +1233,7 @@ pub fn scrollDown(self: *Terminal, count: usize) void {
 
     // Move to the top of the scroll region
     self.screen.cursor.y = self.scrolling_region.top;
-    self.insertLines(count);
+    try self.insertLines(count);
 }
 
 /// Removes amount lines from the top of the scroll region. The remaining lines
@@ -1172,14 +1244,14 @@ pub fn scrollDown(self: *Terminal, count: usize) void {
 ///
 /// Does not change the (absolute) cursor position.
 // TODO: test
-pub fn scrollUp(self: *Terminal, count: usize) void {
+pub fn scrollUp(self: *Terminal, count: usize) !void {
     // Preserve the cursor
     const cursor = self.screen.cursor;
     defer self.screen.cursor = cursor;
 
     // Move to the top of the scroll region
     self.screen.cursor.y = self.scrolling_region.top;
-    self.deleteLines(count);
+    try self.deleteLines(count);
 }
 
 /// Options for scrolling the viewport of the terminal grid.
@@ -1543,7 +1615,7 @@ test "Terminal: deleteLines" {
     try t.print('D');
 
     t.cursorUp(2);
-    t.deleteLines(1);
+    try t.deleteLines(1);
 
     try t.print('E');
     t.carriageReturn();
@@ -1579,7 +1651,7 @@ test "Terminal: deleteLines with scroll region" {
 
     t.setScrollingRegion(1, 3);
     t.setCursorPos(1, 1);
-    t.deleteLines(1);
+    try t.deleteLines(1);
 
     try t.print('E');
     t.carriageReturn();
@@ -1620,7 +1692,7 @@ test "Terminal: insertLines" {
     t.setCursorPos(2, 1);
 
     // Insert two lines
-    t.insertLines(2);
+    try t.insertLines(2);
 
     {
         var str = try t.plainString(testing.allocator);
@@ -1651,7 +1723,7 @@ test "Terminal: insertLines with scroll region" {
 
     t.setScrollingRegion(1, 2);
     t.setCursorPos(1, 1);
-    t.insertLines(1);
+    try t.insertLines(1);
 
     try t.print('X');
 
@@ -1686,7 +1758,7 @@ test "Terminal: insertLines more than remaining" {
     t.setCursorPos(2, 1);
 
     // Insert a bunch of  lines
-    t.insertLines(20);
+    try t.insertLines(20);
 
     {
         var str = try t.plainString(testing.allocator);
@@ -1827,7 +1899,7 @@ test "Terminal: DECALN" {
     t.carriageReturn();
     try t.linefeed();
     try t.print('B');
-    t.decaln();
+    try t.decaln();
 
     try testing.expectEqual(@as(usize, 0), t.screen.cursor.y);
     try testing.expectEqual(@as(usize, 0), t.screen.cursor.x);

src/terminal/color.zig

@@ -95,9 +95,9 @@ pub const Name = enum(u8) {
 
 /// RGB
 pub const RGB = packed struct {
-    r: u8,
+    r: u8 = 0,
-    g: u8,
+    g: u8 = 0,
-    b: u8,
+    b: u8 = 0,
 
     test {
         try std.testing.expectEqual(@as(usize, 3), @sizeOf(RGB));

src/terminal/stream.zig

@@ -51,9 +51,7 @@ pub fn Stream(comptime Handler: type) type {
             const actions = self.parser.next(c);
             for (actions) |action_opt| {
                 // if (action_opt) |action| {
-                //     if (action != .print) {
+                //     log.info("action: {}", .{action});
-                //         log.info("action: {}", .{action});
-                //     }
                 // }
                 switch (action_opt orelse continue) {
                     .print => |p| if (@hasDecl(T, "print")) try self.handler.print(p),