From 60812d418fed45d01278e936abdd12c3e6aef745 Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Fri, 4 Jul 2025 00:50:38 -0400 Subject: [PATCH 01/12] WIP --- build.zig.zon | 5 ++ src/bench/grapheme-break.sh | 2 + src/bench/grapheme-break.zig | 35 +++++++++ src/build/SharedDeps.zig | 12 +++ src/build/UnicodeTables.zig | 10 +-- src/config/Config.zig | 65 +++++++++++++++- src/config/c_get.zig | 10 +++ src/config/formatter.zig | 4 + src/font/CodepointResolver.zig | 11 ++- src/font/SharedGrid.zig | 2 + src/font/SharedGridSet.zig | 8 ++ src/font/shaper/coretext.zig | 42 +++++++++++ src/font/shaper/web_canvas.zig | 8 +- src/global.zig | 48 ++++++++++++ src/input/Binding.zig | 52 +++++++++++-- src/renderer/cell.zig | 12 ++- src/simd/codepoint_width.zig | 54 +++++++++----- src/terminal/Terminal.zig | 6 +- src/unicode/grapheme.zig | 13 ++-- src/unicode/lut.zig | 2 +- src/unicode/main.zig | 1 - src/unicode/props.zig | 132 +++++++++++++++++++-------------- 22 files changed, 427 insertions(+), 107 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 237720f35..e1bf7c814 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -41,6 +41,11 @@ .hash = "ziglyph-0.11.2-AAAAAHPtHwB4Mbzn1KvOV7Wpjo82NYEc_v0WC8oCLrkf", .lazy = true, }, + .zg = .{ + .url = "https://codeberg.org/jacobsandlund/zg/archive/21be4c72f2aff537d7630a7aeb641befc935db89.tar.gz", + .hash = "zg-0.14.0-oGqU3IkvswJcetRSabEnsUe_jsWz48WkZebbVS-pJ5jB", + .lazy = true, + }, .zig_wayland = .{ // codeberg ifreund/zig-wayland .url = "https://codeberg.org/ifreund/zig-wayland/archive/f3c5d503e540ada8cbcb056420de240af0c094f7.tar.gz", diff --git a/src/bench/grapheme-break.sh b/src/bench/grapheme-break.sh index 24f475caa..832728951 100755 --- a/src/bench/grapheme-break.sh +++ b/src/bench/grapheme-break.sh @@ -27,6 +27,8 @@ hyperfine \ "./zig-out/bin/bench-grapheme-break --mode=noop${ARGS} try benchNoop(reader, buf), .ziglyph => try benchZiglyph(reader, buf), + .zg => try benchZg(&graphemes, reader, buf), .table => try benchTable(reader, buf), } } @@ -118,6 +127,32 @@ noinline fn benchTable( } } +noinline fn benchZg( + graphemes: *const Graphemes, + reader: anytype, + buf: []u8, +) !void { + var d: UTF8Decoder = .{}; + var state: Graphemes.State = .{}; + var cp1: u21 = 0; + while (true) { + const n = try reader.read(buf); + if (n == 0) break; + + // Using stream.next directly with a for loop applies a naive + // scalar approach. + for (buf[0..n]) |c| { + const cp_, const consumed = d.next(c); + assert(consumed); + if (cp_) |cp2| { + const v = Graphemes.graphemeBreak(cp1, @intCast(cp2), graphemes, &state); + buf[0] = @intCast(@intFromBool(v)); + cp1 = cp2; + } + } + } +} + noinline fn benchZiglyph( reader: anytype, buf: []u8, diff --git a/src/build/SharedDeps.zig b/src/build/SharedDeps.zig index f173e4856..5ec6664f3 100644 --- a/src/build/SharedDeps.zig +++ b/src/build/SharedDeps.zig @@ -417,6 +417,18 @@ pub fn add( })) |dep| { step.root_module.addImport("ziglyph", dep.module("ziglyph")); } + if (b.lazyDependency("zg", .{ + .target = target, + .optimize = optimize, + })) |dep| { + step.root_module.addImport("CaseFolding", dep.module("CaseFolding")); + // Only needed for table gen, but still include to have build succeeed + step.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); + step.root_module.addImport("Emoji", dep.module("Emoji")); + step.root_module.addImport("GeneralCategories", dep.module("GeneralCategories")); + step.root_module.addImport("Graphemes", dep.module("Graphemes")); + step.root_module.addImport("LetterCasing", dep.module("LetterCasing")); + } if (b.lazyDependency("zf", .{ .target = target, .optimize = optimize, diff --git a/src/build/UnicodeTables.zig b/src/build/UnicodeTables.zig index 5bba2341b..10f2064f7 100644 --- a/src/build/UnicodeTables.zig +++ b/src/build/UnicodeTables.zig @@ -21,13 +21,11 @@ pub fn init(b: *std.Build) !UnicodeTables { }), }); - if (b.lazyDependency("ziglyph", .{ + if (b.lazyDependency("zg", .{ .target = b.graph.host, - })) |ziglyph_dep| { - exe.root_module.addImport( - "ziglyph", - ziglyph_dep.module("ziglyph"), - ); + })) |dep| { + exe.root_module.addImport("Graphemes", dep.module("Graphemes")); + exe.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); } const run = b.addRunArtifact(exe); diff --git a/src/config/Config.zig b/src/config/Config.zig index f36132ea9..7d4155362 100644 --- a/src/config/Config.zig +++ b/src/config/Config.zig @@ -16,7 +16,6 @@ const build_config = @import("../build_config.zig"); const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; -const global_state = &@import("../global.zig").state; const fontpkg = @import("../font/main.zig"); const inputpkg = @import("../input.zig"); const terminal = @import("../terminal/main.zig"); @@ -5765,6 +5764,8 @@ pub const Keybinds = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var set: Keybinds = .{}; try set.parseCLI(alloc, "shift+a=copy_to_clipboard"); @@ -5779,6 +5780,8 @@ pub const Keybinds = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: Keybinds = .{}; try list.parseCLI(alloc, "shift+a=csi:hello"); @@ -5795,6 +5798,8 @@ pub const Keybinds = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: Keybinds = .{}; try list.parseCLI(alloc, "ctrl+z>1=goto_tab:1"); @@ -5819,6 +5824,8 @@ pub const Keybinds = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: Keybinds = .{}; try list.parseCLI(alloc, "ctrl+a>ctrl+b>n=new_window"); @@ -6030,6 +6037,8 @@ pub const RepeatableCodepointMap = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: Self = .{}; try list.parseCLI(alloc, "U+ABCD=Comic Sans"); @@ -6067,6 +6076,8 @@ pub const RepeatableCodepointMap = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: Self = .{}; try list.parseCLI(alloc, "U+ABCD=Comic Sans"); @@ -6082,6 +6093,8 @@ pub const RepeatableCodepointMap = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: Self = .{}; try list.parseCLI(alloc, "U+0001 - U+0005=Verdana"); @@ -6097,6 +6110,8 @@ pub const RepeatableCodepointMap = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: Self = .{}; try list.parseCLI(alloc, "U+0006-U+0009, U+ABCD=Courier"); @@ -6175,6 +6190,8 @@ pub const FontStyle = union(enum) { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var p: Self = .{ .default = {} }; try p.parseCLI(alloc, "default"); @@ -6195,6 +6212,8 @@ pub const FontStyle = union(enum) { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var p: Self = .{ .default = {} }; try p.parseCLI(alloc, "default"); @@ -6210,6 +6229,8 @@ pub const FontStyle = union(enum) { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var p: Self = .{ .default = {} }; try p.parseCLI(alloc, "false"); @@ -6225,6 +6246,8 @@ pub const FontStyle = union(enum) { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var p: Self = .{ .default = {} }; try p.parseCLI(alloc, "bold"); @@ -6402,6 +6425,8 @@ pub const RepeatableCommand = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: RepeatableCommand = .{}; try list.parseCLI(alloc, "title:Foo,action:ignore"); @@ -6447,6 +6472,8 @@ pub const RepeatableCommand = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: RepeatableCommand = .{}; try list.parseCLI(alloc, "title:Bobr, action:text:Bober"); @@ -6462,6 +6489,8 @@ pub const RepeatableCommand = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var list: RepeatableCommand = .{}; try list.parseCLI(alloc, "title:Bobr, action:text:kurwa"); @@ -7180,6 +7209,8 @@ pub const Theme = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); // Single { @@ -7597,6 +7628,8 @@ const TestIterator = struct { test "parse hook: invalid command" { const testing = std.testing; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(testing.allocator); defer cfg.deinit(); const alloc = cfg._arena.?.allocator(); @@ -7608,6 +7641,8 @@ test "parse hook: invalid command" { test "parse e: command only" { const testing = std.testing; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(testing.allocator); defer cfg.deinit(); const alloc = cfg._arena.?.allocator(); @@ -7623,6 +7658,8 @@ test "parse e: command only" { test "parse e: command and args" { const testing = std.testing; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(testing.allocator); defer cfg.deinit(); const alloc = cfg._arena.?.allocator(); @@ -7641,6 +7678,8 @@ test "parse e: command and args" { test "clone default" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var source = try Config.default(alloc); defer source.deinit(); @@ -7658,6 +7697,8 @@ test "clone default" { test "clone preserves conditional state" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var a = try Config.default(alloc); defer a.deinit(); @@ -7686,6 +7727,8 @@ test "clone can then change conditional state" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -7746,6 +7789,8 @@ test "clone can then change conditional state" { test "clone preserves conditional set" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -7765,6 +7810,8 @@ test "clone preserves conditional set" { test "changed" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var source = try Config.default(alloc); defer source.deinit(); @@ -7779,6 +7826,8 @@ test "changed" { test "changeConditionalState ignores irrelevant changes" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); { var cfg = try Config.default(alloc); @@ -7798,6 +7847,8 @@ test "changeConditionalState ignores irrelevant changes" { test "changeConditionalState applies relevant changes" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); { var cfg = try Config.default(alloc); @@ -7820,6 +7871,8 @@ test "theme loading" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -7856,6 +7909,8 @@ test "theme loading preserves conditional state" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -7886,6 +7941,8 @@ test "theme priority is lower than config" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -7920,6 +7977,8 @@ test "theme loading correct light/dark" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -8009,6 +8068,8 @@ test "theme loading correct light/dark" { test "theme specifying light/dark changes window-theme from auto" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); { var cfg = try Config.default(alloc); @@ -8027,6 +8088,8 @@ test "theme specifying light/dark changes window-theme from auto" { test "theme specifying light/dark sets theme usage in conditional state" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); { var cfg = try Config.default(alloc); diff --git a/src/config/c_get.zig b/src/config/c_get.zig index f235f596a..6be62bbc6 100644 --- a/src/config/c_get.zig +++ b/src/config/c_get.zig @@ -121,6 +121,8 @@ fn fieldByKey(self: *const Config, comptime k: Key) Value(k) { test "c_get: u8" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); @@ -134,6 +136,8 @@ test "c_get: u8" { test "c_get: enum" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); @@ -149,6 +153,8 @@ test "c_get: enum" { test "c_get: color" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); @@ -164,6 +170,8 @@ test "c_get: color" { test "c_get: optional" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); @@ -187,6 +195,8 @@ test "c_get: optional" { test "c_get: background-blur" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); diff --git a/src/config/formatter.zig b/src/config/formatter.zig index cabf80953..89298d425 100644 --- a/src/config/formatter.zig +++ b/src/config/formatter.zig @@ -193,6 +193,8 @@ pub const FileFormatter = struct { test "format default config" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -212,6 +214,8 @@ test "format default config" { test "format default config changed" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); cfg.@"font-size" = 42; diff --git a/src/font/CodepointResolver.zig b/src/font/CodepointResolver.zig index 16536300c..5c87bfdb2 100644 --- a/src/font/CodepointResolver.zig +++ b/src/font/CodepointResolver.zig @@ -13,8 +13,9 @@ const CodepointResolver = @This(); const std = @import("std"); const Allocator = std.mem.Allocator; -const ziglyph = @import("ziglyph"); +const Emoji = @import("Emoji"); const font = @import("main.zig"); +const zg = &@import("../global.zig").state.zg; const Atlas = font.Atlas; const CodepointMap = font.CodepointMap; const Collection = font.Collection; @@ -150,7 +151,7 @@ pub fn getIndex( // we'll do this multiple times if we recurse, but this is a cached function // call higher up (GroupCache) so this should be rare. const p_mode: Collection.PresentationMode = if (p) |v| .{ .explicit = v } else .{ - .default = if (ziglyph.emoji.isEmojiPresentation(@intCast(cp))) + .default = if (Emoji.isEmojiPresentation(zg.emoji, @intCast(cp))) .emoji else .text, @@ -379,6 +380,8 @@ test getIndex { const testFont = font.embedded.regular; const testEmoji = font.embedded.emoji; const testEmojiText = font.embedded.emoji_text; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var lib = try Library.init(alloc); defer lib.deinit(); @@ -457,6 +460,8 @@ test "getIndex disabled font style" { const testing = std.testing; const alloc = testing.allocator; const testFont = font.embedded.regular; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var atlas_grayscale = try font.Atlas.init(alloc, 512, .grayscale); defer atlas_grayscale.deinit(alloc); @@ -512,6 +517,8 @@ test "getIndex disabled font style" { test "getIndex box glyph" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var lib = try Library.init(alloc); defer lib.deinit(); diff --git a/src/font/SharedGrid.zig b/src/font/SharedGrid.zig index dcfa0a551..f24a7d816 100644 --- a/src/font/SharedGrid.zig +++ b/src/font/SharedGrid.zig @@ -369,6 +369,8 @@ fn testGrid(mode: TestMode, alloc: Allocator, lib: Library) !SharedGrid { test getIndex { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); // const testEmoji = @import("test.zig").fontEmoji; var lib = try Library.init(alloc); diff --git a/src/font/SharedGridSet.zig b/src/font/SharedGridSet.zig index e3e61907b..bba2d3e0c 100644 --- a/src/font/SharedGridSet.zig +++ b/src/font/SharedGridSet.zig @@ -694,6 +694,8 @@ pub const Key = struct { test "Key" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -713,6 +715,8 @@ test "Key" { test "Key different font points" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -731,6 +735,8 @@ test "Key different font points" { test "Key different font DPI" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -749,6 +755,8 @@ test "Key different font DPI" { test SharedGridSet { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var set = try SharedGridSet.init(alloc); defer set.deinit(); diff --git a/src/font/shaper/coretext.zig b/src/font/shaper/coretext.zig index 1aaa029dc..ae663ea39 100644 --- a/src/font/shaper/coretext.zig +++ b/src/font/shaper/coretext.zig @@ -576,6 +576,8 @@ pub const Shaper = struct { test "run iterator" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -656,6 +658,8 @@ test "run iterator" { test "run iterator: empty cells with background set" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -704,6 +708,8 @@ test "run iterator: empty cells with background set" { test "shape" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -737,6 +743,8 @@ test "shape" { test "shape nerd fonts" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .nerd_font); defer testdata.deinit(); @@ -770,6 +778,8 @@ test "shape nerd fonts" { test "shape inconsolata ligs" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -824,6 +834,8 @@ test "shape inconsolata ligs" { test "shape monaspace ligs" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .monaspace_neon); defer testdata.deinit(); @@ -856,6 +868,8 @@ test "shape monaspace ligs" { test "shape left-replaced lig in last run" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .geist_mono); defer testdata.deinit(); @@ -888,6 +902,8 @@ test "shape left-replaced lig in last run" { test "shape left-replaced lig in early run" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .geist_mono); defer testdata.deinit(); @@ -917,6 +933,8 @@ test "shape left-replaced lig in early run" { test "shape U+3C9 with JB Mono" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .jetbrains_mono); defer testdata.deinit(); @@ -948,6 +966,8 @@ test "shape U+3C9 with JB Mono" { test "shape emoji width" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -977,6 +997,8 @@ test "shape emoji width" { test "shape emoji width long" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1024,6 +1046,8 @@ test "shape emoji width long" { test "shape variation selector VS15" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1057,6 +1081,8 @@ test "shape variation selector VS15" { test "shape variation selector VS16" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1090,6 +1116,8 @@ test "shape variation selector VS16" { test "shape with empty cells in between" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1121,6 +1149,8 @@ test "shape with empty cells in between" { test "shape Chinese characters" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1161,6 +1191,8 @@ test "shape Chinese characters" { test "shape box glyphs" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1198,6 +1230,8 @@ test "shape box glyphs" { test "shape selection boundary" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1321,6 +1355,8 @@ test "shape selection boundary" { test "shape cursor boundary" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1458,6 +1494,8 @@ test "shape cursor boundary" { test "shape cursor boundary and colored emoji" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1552,6 +1590,8 @@ test "shape cursor boundary and colored emoji" { test "shape cell attribute change" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1680,6 +1720,8 @@ test "shape high plane sprite font codepoint" { const testing = std.testing; const alloc = testing.allocator; + const zg = try @import("../../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); diff --git a/src/font/shaper/web_canvas.zig b/src/font/shaper/web_canvas.zig index 4ed4b7db6..f6b39c7cd 100644 --- a/src/font/shaper/web_canvas.zig +++ b/src/font/shaper/web_canvas.zig @@ -1,9 +1,10 @@ const std = @import("std"); const assert = std.debug.assert; const Allocator = std.mem.Allocator; -const ziglyph = @import("ziglyph"); +const Graphemes = @import("Graphemes"); const font = @import("../main.zig"); const terminal = @import("../../terminal/main.zig"); +const zg = &@import("../../global.zig").state.zg; const log = std.log.scoped(.font_shaper); @@ -111,7 +112,7 @@ pub const Shaper = struct { // font ligatures. However, we do support grapheme clustering. // This means we can render things like skin tone emoji but // we can't render things like single glyph "=>". - var break_state: u3 = 0; + var break_state: Graphemes.State = .{}; var cp1: u21 = @intCast(codepoints[0]); var start: usize = 0; @@ -126,9 +127,10 @@ pub const Shaper = struct { const cp2: u21 = @intCast(codepoints[i]); defer cp1 = cp2; - break :blk ziglyph.graphemeBreak( + break :blk Graphemes.graphemeBreak( cp1, cp2, + zg.graphemes, &break_state, ); }; diff --git a/src/global.zig b/src/global.zig index 668d2faec..cdd3541dc 100644 --- a/src/global.zig +++ b/src/global.zig @@ -10,6 +10,12 @@ const oni = @import("oniguruma"); const crash = @import("crash/main.zig"); const renderer = @import("renderer.zig"); const apprt = @import("apprt.zig"); +const CaseFolding = @import("CaseFolding"); +const Emoji = @import("Emoji"); +const GeneralCategories = @import("GeneralCategories"); +const Graphemes = @import("Graphemes"); +const LetterCasing = @import("LetterCasing"); +const unicode = @import("unicode/main.zig"); /// We export the xev backend we want to use so that the rest of /// Ghostty can import this once and have access to the proper @@ -33,6 +39,7 @@ pub const GlobalState = struct { action: ?cli.Action, logging: Logging, rlimits: ResourceLimits = .{}, + zg: Zg, /// The app resources directory, equivalent to zig-out/share when we build /// from source. This is null if we can't detect it. @@ -64,6 +71,7 @@ pub const GlobalState = struct { .logging = .{ .stderr = {} }, .rlimits = .{}, .resources_dir = .{}, + .zg = undefined, }; errdefer self.deinit(); @@ -150,6 +158,8 @@ pub const GlobalState = struct { ); }; + self.zg = try Zg.init(self.alloc); + // const sentrylib = @import("sentry"); // if (sentrylib.captureEvent(sentrylib.Value.initMessageEvent( // .info, @@ -188,6 +198,8 @@ pub const GlobalState = struct { // Flush our crash logs crash.deinit(); + self.zg.deinit(self.alloc); + if (self.gpa) |*value| { // We want to ensure that we deinit the GPA because this is // the point at which it will output if there were safety violations. @@ -216,6 +228,42 @@ pub const GlobalState = struct { } }; +pub const Zg = struct { + case_folding: CaseFolding, + emoji: Emoji, + general_categories: GeneralCategories, + graphemes: Graphemes, + letter_casing: LetterCasing, + + pub fn init(alloc: std.mem.Allocator) !Zg { + return .{ + .case_folding = try CaseFolding.init(alloc), + .emoji = try Emoji.init(alloc), + .general_categories = try GeneralCategories.init(alloc), + .graphemes = try Graphemes.init(alloc), + .letter_casing = try LetterCasing.init(alloc), + }; + } + + pub fn deinit(self: *Zg, alloc: std.mem.Allocator) void { + self.case_folding.deinit(alloc); + self.emoji.deinit(alloc); + self.general_categories.deinit(alloc); + self.graphemes.deinit(alloc); + self.letter_casing.deinit(alloc); + } + + pub fn initForTesting() !*Zg { + const zg = &state.zg; + zg.* = try Zg.init(std.testing.allocator); + return zg; + } + + pub fn deinitForTesting(self: *Zg) void { + self.deinit(std.testing.allocator); + } +}; + /// Maintains the Unix resource limits that we set for our process. This /// can be used to restore the limits to their original values. pub const ResourceLimits = struct { diff --git a/src/input/Binding.zig b/src/input/Binding.zig index 7cdb8047c..d87ac12c0 100644 --- a/src/input/Binding.zig +++ b/src/input/Binding.zig @@ -5,9 +5,11 @@ const Binding = @This(); const std = @import("std"); const Allocator = std.mem.Allocator; const assert = std.debug.assert; -const ziglyph = @import("ziglyph"); +const LetterCasing = @import("LetterCasing"); +const CaseFolding = @import("CaseFolding"); const key = @import("key.zig"); const KeyEvent = key.KeyEvent; +const zg = &@import("../global.zig").state.zg; /// The trigger that needs to be performed to execute the action. trigger: Trigger, @@ -1551,15 +1553,17 @@ pub const Trigger = struct { /// in more codepoints so we need to use a 3 element array. fn foldedCodepoint(cp: u21) [3]u21 { // ASCII fast path - if (ziglyph.letter.isAsciiLetter(cp)) { - return .{ ziglyph.letter.toLower(cp), 0, 0 }; + if (('A' <= cp and cp <= 'Z') or ('a' <= cp and cp <= 'z')) { + return .{ LetterCasing.toLower(zg.letter_casing, cp), 0, 0 }; } - // Unicode slow path. Case folding can resultin more codepoints. + // Unicode slow path. Case folding can result in more codepoints. // If more codepoints are produced then we return the codepoint // as-is which isn't correct but until we have a failing test // then I don't want to handle this. - return ziglyph.letter.toCaseFold(cp); + var buf: [3]u21 = .{ 0, 0, 0 }; + _ = CaseFolding.caseFold(&zg.case_folding, cp, &buf); + return buf; } /// Convert the trigger to a C API compatible trigger. @@ -2612,6 +2616,8 @@ test "parse: sequences" { test "set: parseAndPut typical binding" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2635,6 +2641,8 @@ test "set: parseAndPut typical binding" { test "set: parseAndPut unconsumed binding" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2659,6 +2667,8 @@ test "set: parseAndPut unconsumed binding" { test "set: parseAndPut removed binding" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2677,6 +2687,8 @@ test "set: parseAndPut removed binding" { test "set: parseAndPut sequence" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2701,6 +2713,8 @@ test "set: parseAndPut sequence" { test "set: parseAndPut sequence with two actions" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2733,6 +2747,8 @@ test "set: parseAndPut sequence with two actions" { test "set: parseAndPut overwrite sequence" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2758,6 +2774,8 @@ test "set: parseAndPut overwrite sequence" { test "set: parseAndPut overwrite leader" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2783,6 +2801,8 @@ test "set: parseAndPut overwrite leader" { test "set: parseAndPut unbind sequence unbinds leader" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2799,6 +2819,8 @@ test "set: parseAndPut unbind sequence unbinds leader" { test "set: parseAndPut unbind sequence unbinds leader if not set" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2814,6 +2836,8 @@ test "set: parseAndPut unbind sequence unbinds leader if not set" { test "set: parseAndPut sequence preserves reverse mapping" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2831,6 +2855,8 @@ test "set: parseAndPut sequence preserves reverse mapping" { test "set: put overwrites sequence" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2851,6 +2877,8 @@ test "set: put overwrites sequence" { test "set: maintains reverse mapping" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2879,6 +2907,8 @@ test "set: maintains reverse mapping" { test "set: performable is not part of reverse mappings" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2912,6 +2942,8 @@ test "set: performable is not part of reverse mappings" { test "set: overriding a mapping updates reverse" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2933,6 +2965,8 @@ test "set: overriding a mapping updates reverse" { test "set: consumed state" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2958,6 +2992,8 @@ test "set: consumed state" { test "set: getEvent physical" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2988,6 +3024,8 @@ test "set: getEvent physical" { test "set: getEvent codepoint" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -3028,6 +3066,8 @@ test "set: getEvent codepoint" { test "set: getEvent codepoint case folding" { const testing = std.testing; const alloc = testing.allocator; + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -3070,6 +3110,8 @@ test "Action: clone" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); + _ = try @import("../global.zig").Zg.initForTesting(); + defer zg.deinitForTesting(); { var a: Action = .ignore; diff --git a/src/renderer/cell.zig b/src/renderer/cell.zig index ef7122699..39ddc6ee1 100644 --- a/src/renderer/cell.zig +++ b/src/renderer/cell.zig @@ -1,12 +1,13 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const assert = std.debug.assert; -const ziglyph = @import("ziglyph"); const font = @import("../font/main.zig"); const terminal = @import("../terminal/main.zig"); const renderer = @import("../renderer.zig"); const shaderpkg = renderer.Renderer.API.shaders; const ArrayListCollection = @import("../datastruct/array_list_collection.zig").ArrayListCollection; +const GeneralCategories = @import("GeneralCategories"); +const zg = &@import("../global.zig").state.zg; /// The possible cell content keys that exist. pub const Key = enum { @@ -246,8 +247,10 @@ pub fn fgMode( const cell = cell_pin.rowAndCell().cell; const cp = cell.codepoint(); - if (!ziglyph.general_category.isPrivateUse(cp) and - !ziglyph.blocks.isDingbats(cp)) + // If it's not Private Use (Co) or Dingbats (0x2700-0x27bf), use + // normal mode. + if (GeneralCategories.gc(zg.general_categories, cp) != .Co and + !(cp >= 0x2700 and cp <= 0x27bf)) { break :text .normal; } @@ -278,7 +281,8 @@ pub fn fgMode( // Powerline is whitespace if (isPowerline(prev_cp)) break :prev; - if (ziglyph.general_category.isPrivateUse(prev_cp)) { + // If it's Private Use (Co), then we use constrained mode. + if (GeneralCategories.gc(zg.general_categories, cp) == .Co) { break :text .constrained; } } diff --git a/src/simd/codepoint_width.zig b/src/simd/codepoint_width.zig index aab4bdd95..e94b9a300 100644 --- a/src/simd/codepoint_width.zig +++ b/src/simd/codepoint_width.zig @@ -4,7 +4,9 @@ const std = @import("std"); extern "c" fn ghostty_simd_codepoint_width(u32) i8; pub fn codepointWidth(cp: u32) i8 { - //return @import("ziglyph").display_width.codePointWidth(@intCast(cp), .half); + // const zg = try @import("../../global.zig").Zg.initForTesting(); + // defer zg.deinitForTesting(); + // try testing.expectEqual(@as(i8, 1), @import("DisplayWidth").codePointWidth(zg.display_width, @intCast(cp))); return ghostty_simd_codepoint_width(cp); } @@ -19,27 +21,41 @@ test "codepointWidth basic" { try testing.expectEqual(@as(i8, 2), codepointWidth(0xF900)); // 豈 try testing.expectEqual(@as(i8, 2), codepointWidth(0x20000)); // 𠀀 try testing.expectEqual(@as(i8, 2), codepointWidth(0x30000)); // 𠀀 - // try testing.expectEqual(@as(i8, 1), @import("ziglyph").display_width.codePointWidth(0x100, .half)); + // const zg = try @import("../../global.zig").Zg.initForTesting(); + // defer zg.deinitForTesting(); + // try testing.expectEqual(@as(i8, 1), @import("DisplayWidth").codePointWidth(zg.display_width, 0x100)); } // This is not very fast in debug modes, so its commented by default. // IMPORTANT: UNCOMMENT THIS WHENEVER MAKING CODEPOINTWIDTH CHANGES. -// test "codepointWidth matches ziglyph" { -// const testing = std.testing; -// const ziglyph = @import("ziglyph"); +//test "codepointWidth matches zg" { +// const testing = std.testing; +// const DisplayWidth = @import("DisplayWidth"); +// const display_width = try DisplayWidth.init(std.testing.allocator); +// defer display_width.deinit(std.testing.allocator); +// var success: bool = true; // -// const min = 0xFF + 1; // start outside ascii -// for (min..std.math.maxInt(u21)) |cp| { -// const simd = codepointWidth(@intCast(cp)); -// const zg = ziglyph.display_width.codePointWidth(@intCast(cp), .half); -// if (simd != zg) mismatch: { -// if (cp == 0x2E3B) { -// try testing.expectEqual(@as(i8, 2), simd); -// break :mismatch; -// } +// const min = 0xFF + 1; // start outside ascii +// for (min..0x110000) |cp| { +// const simd = codepointWidth(@intCast(cp)); +// const zg_width = DisplayWidth.codePointWidth(display_width, @intCast(cp)); +// if (simd != zg_width) mismatch: { +// if (cp == 0x2E3B) { +// try testing.expectEqual(@as(i8, 2), simd); +// std.log.warn("mismatch for 0x2e3b cp=U+{x} simd={} zg={}", .{ cp, simd, zg_width }); +// break :mismatch; +// } // -// std.log.warn("mismatch cp=U+{x} simd={} zg={}", .{ cp, simd, zg }); -// try testing.expect(false); -// } -// } -// } +// if (cp == 0x890) { +// try testing.expectEqual(@as(i8, 0), simd); +// try testing.expectEqual(@as(i8, 1), zg_width); +// break :mismatch; +// } +// +// std.log.warn("mismatch cp=U+{x} simd={} zg={}", .{ cp, simd, zg_width }); +// success = false; +// } +// } +// +// try testing.expect(success); +//} diff --git a/src/terminal/Terminal.zig b/src/terminal/Terminal.zig index dd7207f6d..11ac1346e 100644 --- a/src/terminal/Terminal.zig +++ b/src/terminal/Terminal.zig @@ -344,7 +344,7 @@ pub fn print(self: *Terminal, c: u21) !void { // VS15 makes it narrow. if (c == 0xFE0F or c == 0xFE0E) { // This only applies to emoji - const prev_props = unicode.getProperties(prev.cell.content.codepoint); + const prev_props = unicode.table.get(prev.cell.content.codepoint); const emoji = prev_props.grapheme_boundary_class.isExtendedPictographic(); if (!emoji) return; @@ -416,7 +416,7 @@ pub fn print(self: *Terminal, c: u21) !void { const width: usize = if (c <= 0xFF) 1 else @intCast(unicode.table.get(c).width); // Note: it is possible to have a width of "3" and a width of "-1" - // from ziglyph. We should look into those cases and handle them + // from zg. We should look into those cases and handle them // appropriately. assert(width <= 2); // log.debug("c={x} width={}", .{ c, width }); @@ -452,7 +452,7 @@ pub fn print(self: *Terminal, c: u21) !void { // If this is a emoji variation selector, prev must be an emoji if (c == 0xFE0F or c == 0xFE0E) { - const prev_props = unicode.getProperties(prev.content.codepoint); + const prev_props = unicode.table.get(prev.content.codepoint); const emoji = prev_props.grapheme_boundary_class == .extended_pictographic; if (!emoji) return; } diff --git a/src/unicode/grapheme.zig b/src/unicode/grapheme.zig index 7847ef6f5..2ba36161f 100644 --- a/src/unicode/grapheme.zig +++ b/src/unicode/grapheme.zig @@ -149,21 +149,19 @@ fn graphemeBreakClass( return true; } -/// If you build this file as a binary, we will verify the grapheme break -/// implementation. This iterates over billions of codepoints so it is -/// SLOW. It's not meant to be run in CI, but it's useful for debugging. -pub fn main() !void { +// This test will verify the grapheme break implementation. This iterates over billions of codepoints so it is SLOW. +// It's not meant to be run in CI, but it's useful for debugging. +test "grapheme break check against ziglyph" { const ziglyph = @import("ziglyph"); // Set the min and max to control the test range. const min = 0; const max = std.math.maxInt(u21) + 1; + var success: bool = true; var state: BreakState = .{}; var zg_state: u3 = 0; for (min..max) |cp1| { - if (cp1 % 1000 == 0) std.log.warn("progress cp1={}", .{cp1}); - if (cp1 == '\r' or cp1 == '\n' or ziglyph.grapheme_break.isControl(@intCast(cp1))) continue; @@ -174,6 +172,7 @@ pub fn main() !void { const gb = graphemeBreak(@intCast(cp1), @intCast(cp2), &state); const zg_gb = ziglyph.graphemeBreak(@intCast(cp1), @intCast(cp2), &zg_state); if (gb != zg_gb) { + success = false; std.log.warn("cp1={x} cp2={x} gb={} state={} zg_gb={} zg_state={}", .{ cp1, cp2, @@ -185,6 +184,8 @@ pub fn main() !void { } } } + + try std.testing.expect(success); } pub const std_options = struct { diff --git a/src/unicode/lut.zig b/src/unicode/lut.zig index 95c6a3688..84d51792d 100644 --- a/src/unicode/lut.zig +++ b/src/unicode/lut.zig @@ -11,7 +11,7 @@ const Allocator = std.mem.Allocator; /// can in theory be generated at runtime. /// /// Context must have two functions: -/// - `get(Context, u21) Elem`: returns the mapping for a given codepoint +/// - `get(Context, u21) !Elem`: returns the mapping for a given codepoint /// - `eql(Context, Elem, Elem) bool`: returns true if two mappings are equal /// pub fn Generator( diff --git a/src/unicode/main.zig b/src/unicode/main.zig index f5b911948..e8ba05b72 100644 --- a/src/unicode/main.zig +++ b/src/unicode/main.zig @@ -4,7 +4,6 @@ const grapheme = @import("grapheme.zig"); const props = @import("props.zig"); pub const table = props.table; pub const Properties = props.Properties; -pub const getProperties = props.get; pub const graphemeBreak = grapheme.graphemeBreak; pub const GraphemeBreakState = grapheme.BreakState; diff --git a/src/unicode/props.zig b/src/unicode/props.zig index 99c57aa0a..c4c147c70 100644 --- a/src/unicode/props.zig +++ b/src/unicode/props.zig @@ -1,9 +1,26 @@ const props = @This(); const std = @import("std"); const assert = std.debug.assert; -const ziglyph = @import("ziglyph"); +const Graphemes = @import("Graphemes"); +const DisplayWidth = @import("DisplayWidth"); const lut = @import("lut.zig"); +graphemes: Graphemes, +display_width: DisplayWidth, + +fn init(alloc: std.mem.Allocator) !props { + const graphemes = try Graphemes.init(alloc); + return .{ + .graphemes = graphemes, + .display_width = try DisplayWidth.initWithGraphemes(alloc, graphemes), + }; +} + +fn deinit(self: *props, alloc: std.mem.Allocator) void { + self.graphemes.deinit(alloc); + self.display_width.deinit(alloc); +} + /// The lookup tables for Ghostty. pub const table = table: { // This is only available after running main() below as part of the Ghostty @@ -79,31 +96,26 @@ pub const GraphemeBoundaryClass = enum(u4) { /// Gets the grapheme boundary class for a codepoint. This is VERY /// SLOW. The use case for this is only in generating lookup tables. - pub fn init(cp: u21) GraphemeBoundaryClass { - // We special-case modifier bases because we should not break - // if a modifier isn't next to a base. - if (ziglyph.emoji.isEmojiModifierBase(cp)) { - assert(ziglyph.emoji.isExtendedPictographic(cp)); - return .extended_pictographic_base; - } - - if (ziglyph.emoji.isEmojiModifier(cp)) return .emoji_modifier; - if (ziglyph.emoji.isExtendedPictographic(cp)) return .extended_pictographic; - if (ziglyph.grapheme_break.isL(cp)) return .L; - if (ziglyph.grapheme_break.isV(cp)) return .V; - if (ziglyph.grapheme_break.isT(cp)) return .T; - if (ziglyph.grapheme_break.isLv(cp)) return .LV; - if (ziglyph.grapheme_break.isLvt(cp)) return .LVT; - if (ziglyph.grapheme_break.isPrepend(cp)) return .prepend; - if (ziglyph.grapheme_break.isExtend(cp)) return .extend; - if (ziglyph.grapheme_break.isZwj(cp)) return .zwj; - if (ziglyph.grapheme_break.isSpacingmark(cp)) return .spacing_mark; - if (ziglyph.grapheme_break.isRegionalIndicator(cp)) return .regional_indicator; - - // This is obviously not INVALID invalid, there is SOME grapheme - // boundary class for every codepoint. But we don't care about - // anything that doesn't fit into the above categories. - return .invalid; + pub fn init(ctx: props, cp: u21) GraphemeBoundaryClass { + return switch (Graphemes.gbp(ctx.graphemes, cp)) { + .Emoji_Modifier_Base => .extended_pictographic_base, + .Emoji_Modifier => .emoji_modifier, + .Extended_Pictographic => .extended_pictographic, + .L => .L, + .V => .V, + .T => .T, + .LV => .LV, + .LVT => .LVT, + .Prepend => .prepend, + .Extend => .extend, + .ZWJ => .zwj, + .SpacingMark => .spacing_mark, + .Regional_Indicator => .regional_indicator, + // This is obviously not INVALID invalid, there is SOME grapheme + // boundary class for every codepoint. But we don't care about + // anything that doesn't fit into the above categories. + .none, .Control, .CR, .LF => .invalid, + }; } /// Returns true if this is an extended pictographic type. This @@ -120,13 +132,25 @@ pub const GraphemeBoundaryClass = enum(u4) { } }; -pub fn get(cp: u21) Properties { - const zg_width = ziglyph.display_width.codePointWidth(cp, .half); +pub fn get(ctx: props, cp: u21) !Properties { + if (cp > 0x10FFFF) { + return .{ + .width = 0, + .grapheme_boundary_class = .invalid, + }; + } else { + const zg_width = DisplayWidth.codePointWidth(ctx.display_width, cp); - return .{ - .width = @intCast(@min(2, @max(0, zg_width))), - .grapheme_boundary_class = .init(cp), - }; + return .{ + .width = @intCast(@min(2, @max(0, zg_width))), + .grapheme_boundary_class = .init(ctx, cp), + }; + } +} + +pub fn eql(ctx: props, a: Properties, b: Properties) bool { + _ = ctx; + return a.eql(b); } /// Runnable binary to generate the lookup tables and output to stdout. @@ -135,20 +159,13 @@ pub fn main() !void { defer arena_state.deinit(); const alloc = arena_state.allocator(); + var self = try init(alloc); + defer self.deinit(alloc); + const gen: lut.Generator( Properties, - struct { - pub fn get(ctx: @This(), cp: u21) !Properties { - _ = ctx; - return props.get(cp); - } - - pub fn eql(ctx: @This(), a: Properties, b: Properties) bool { - _ = ctx; - return a.eql(b); - } - }, - ) = .{}; + props, + ) = .{ .ctx = self }; const t = try gen.generate(alloc); defer alloc.free(t.stage1); @@ -166,16 +183,19 @@ pub fn main() !void { // This is not very fast in debug modes, so its commented by default. // IMPORTANT: UNCOMMENT THIS WHENEVER MAKING CODEPOINTWIDTH CHANGES. -// test "tables match ziglyph" { -// const testing = std.testing; +//test "tables match zg" { +// const testing = std.testing; // -// const min = 0xFF + 1; // start outside ascii -// for (min..std.math.maxInt(u21)) |cp| { -// const t = table.get(@intCast(cp)); -// const zg = @min(2, @max(0, ziglyph.display_width.codePointWidth(@intCast(cp), .half))); -// if (t.width != zg) { -// std.log.warn("mismatch cp=U+{x} t={} zg={}", .{ cp, t, zg }); -// try testing.expect(false); -// } -// } -// } +// const display_width = try DisplayWidth.init(std.testing.allocator); +// defer display_width.deinit(std.testing.allocator); +// +// const min = 0xFF + 1; // start outside ascii +// for (min..0x110000) |cp| { +// const t = table.get(@intCast(cp)); +// const zg = @min(2, @max(0, DisplayWidth.codePointWidth(display_width, @intCast(cp)))); +// if (t.width != zg) { +// std.log.warn("mismatch cp=U+{x} t={} zg={}", .{ cp, t, zg }); +// try testing.expect(false); +// } +// } +//} From 6f9b332e1a96ed58ae30fad45b83134aebc34e17 Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Fri, 4 Jul 2025 12:23:38 -0400 Subject: [PATCH 02/12] Add unicode-test and fix benchmarks --- build.zig | 4 ++ src/bench/codepoint-width.sh | 2 + src/bench/codepoint-width.zig | 19 ++++-- src/bench/grapheme-break.sh | 2 - src/bench/grapheme-break.zig | 32 +-------- src/build/Config.zig | 9 +++ src/build/GhosttyUnicodeTest.zig | 47 +++++++++++++ src/build/SharedDeps.zig | 6 -- src/build/main.zig | 1 + src/unicode/grapheme.zig | 43 ------------ src/unicode/main.zig | 111 +++++++++++++++++++++++++++++++ src/unicode/props.zig | 25 ++----- 12 files changed, 191 insertions(+), 110 deletions(-) create mode 100644 src/build/GhosttyUnicodeTest.zig diff --git a/build.zig b/build.zig index 80af88488..f020a0106 100644 --- a/build.zig +++ b/build.zig @@ -32,6 +32,10 @@ pub fn build(b: *std.Build) !void { const bench = try buildpkg.GhosttyBench.init(b, &deps); if (config.emit_bench) bench.install(); + // Ghostty unicode test exe + const unicode_test = try buildpkg.GhosttyUnicodeTest.init(b, &config, &deps); + if (config.emit_unicode_test) unicode_test.install(); + // Ghostty dist tarball const dist = try buildpkg.GhosttyDist.init(b, &config); { diff --git a/src/bench/codepoint-width.sh b/src/bench/codepoint-width.sh index 43304ec2e..d894f641a 100755 --- a/src/bench/codepoint-width.sh +++ b/src/bench/codepoint-width.sh @@ -27,6 +27,8 @@ hyperfine \ "./zig-out/bin/bench-codepoint-width --mode=noop${ARGS} try benchNoop(reader, buf), .wcwidth => try benchWcwidth(reader, buf), - .ziglyph => try benchZiglyph(reader, buf), + .zg => try benchZg(display_width, reader, buf), .simd => try benchSimd(reader, buf), .table => try benchTable(reader, buf), } @@ -155,7 +159,8 @@ noinline fn benchTable( } } -noinline fn benchZiglyph( +noinline fn benchZg( + display_width: DisplayWidth, reader: anytype, buf: []u8, ) !void { @@ -170,7 +175,7 @@ noinline fn benchZiglyph( const cp_, const consumed = d.next(c); assert(consumed); if (cp_) |cp| { - const width = ziglyph.display_width.codePointWidth(cp, .half); + const width = DisplayWidth.codePointWidth(display_width, cp); // Write the width to the buffer to avoid it being compiled away buf[0] = @intCast(width); diff --git a/src/bench/grapheme-break.sh b/src/bench/grapheme-break.sh index 832728951..02a787b2e 100755 --- a/src/bench/grapheme-break.sh +++ b/src/bench/grapheme-break.sh @@ -25,8 +25,6 @@ hyperfine \ --warmup 10 \ -n noop \ "./zig-out/bin/bench-grapheme-break --mode=noop${ARGS} try benchNoop(reader, buf), - .ziglyph => try benchZiglyph(reader, buf), .zg => try benchZg(&graphemes, reader, buf), .table => try benchTable(reader, buf), } @@ -152,28 +147,3 @@ noinline fn benchZg( } } } - -noinline fn benchZiglyph( - reader: anytype, - buf: []u8, -) !void { - var d: UTF8Decoder = .{}; - var state: u3 = 0; - var cp1: u21 = 0; - while (true) { - const n = try reader.read(buf); - if (n == 0) break; - - // Using stream.next directly with a for loop applies a naive - // scalar approach. - for (buf[0..n]) |c| { - const cp_, const consumed = d.next(c); - assert(consumed); - if (cp_) |cp2| { - const v = ziglyph.graphemeBreak(cp1, @intCast(cp2), &state); - buf[0] = @intCast(@intFromBool(v)); - cp1 = cp2; - } - } - } -} diff --git a/src/build/Config.zig b/src/build/Config.zig index 5f8780af9..90945a1c0 100644 --- a/src/build/Config.zig +++ b/src/build/Config.zig @@ -50,6 +50,7 @@ patch_rpath: ?[]const u8 = null, flatpak: bool = false, emit_test_exe: bool = false, emit_bench: bool = false, +emit_unicode_test: bool = false, emit_helpgen: bool = false, emit_docs: bool = false, emit_webdata: bool = false, @@ -276,6 +277,12 @@ pub fn init(b: *std.Build) !Config { "Build and install the benchmark executables.", ) orelse false; + config.emit_unicode_test = b.option( + bool, + "emit-unicode-test", + "Build and install the unicode test executable.", + ) orelse false; + config.emit_helpgen = b.option( bool, "emit-helpgen", @@ -289,6 +296,7 @@ pub fn init(b: *std.Build) !Config { ) orelse emit_docs: { // If we are emitting any other artifacts then we default to false. if (config.emit_bench or + config.emit_unicode_test or config.emit_test_exe or config.emit_helpgen) break :emit_docs false; @@ -337,6 +345,7 @@ pub fn init(b: *std.Build) !Config { target.result.os.tag == .macos and config.app_runtime == .none and (!config.emit_bench and + !config.emit_unicode_test and !config.emit_test_exe and !config.emit_helpgen); diff --git a/src/build/GhosttyUnicodeTest.zig b/src/build/GhosttyUnicodeTest.zig new file mode 100644 index 000000000..db3575c79 --- /dev/null +++ b/src/build/GhosttyUnicodeTest.zig @@ -0,0 +1,47 @@ +const UnicodeTest = @This(); + +const std = @import("std"); +const Config = @import("Config.zig"); +const SharedDeps = @import("SharedDeps.zig"); + +/// The unicode test executable. +exe: *std.Build.Step.Compile, + +/// The install step for the executable. +install_step: *std.Build.Step.InstallArtifact, + +pub fn init(b: *std.Build, cfg: *const Config, deps: *const SharedDeps) !UnicodeTest { + const exe: *std.Build.Step.Compile = b.addExecutable(.{ + .name = "unicode-test", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/unicode/main.zig"), + .target = cfg.target, + .optimize = cfg.optimize, + .strip = cfg.strip, + .omit_frame_pointer = cfg.strip, + .unwind_tables = if (cfg.strip) .none else .sync, + }), + }); + const install_step = b.addInstallArtifact(exe, .{}); + + // Add the shared dependencies + _ = try deps.add(exe); + + if (b.lazyDependency("ziglyph", .{ + .target = cfg.target, + .optimize = cfg.optimize, + })) |dep| { + exe.root_module.addImport("ziglyph", dep.module("ziglyph")); + } + + return .{ + .exe = exe, + .install_step = install_step, + }; +} + +/// Add the unicode test exe to the install target. +pub fn install(self: *const UnicodeTest) void { + const b = self.install_step.step.owner; + b.getInstallStep().dependOn(&self.install_step.step); +} diff --git a/src/build/SharedDeps.zig b/src/build/SharedDeps.zig index 5ec6664f3..909f727d3 100644 --- a/src/build/SharedDeps.zig +++ b/src/build/SharedDeps.zig @@ -411,12 +411,6 @@ pub fn add( })) |dep| { step.root_module.addImport("z2d", dep.module("z2d")); } - if (b.lazyDependency("ziglyph", .{ - .target = target, - .optimize = optimize, - })) |dep| { - step.root_module.addImport("ziglyph", dep.module("ziglyph")); - } if (b.lazyDependency("zg", .{ .target = target, .optimize = optimize, diff --git a/src/build/main.zig b/src/build/main.zig index 3154d395f..aa1e57827 100644 --- a/src/build/main.zig +++ b/src/build/main.zig @@ -15,6 +15,7 @@ pub const GhosttyFrameData = @import("GhosttyFrameData.zig"); pub const GhosttyLib = @import("GhosttyLib.zig"); pub const GhosttyResources = @import("GhosttyResources.zig"); pub const GhosttyI18n = @import("GhosttyI18n.zig"); +pub const GhosttyUnicodeTest = @import("GhosttyUnicodeTest.zig"); pub const GhosttyXCFramework = @import("GhosttyXCFramework.zig"); pub const GhosttyWebdata = @import("GhosttyWebdata.zig"); pub const HelpStrings = @import("HelpStrings.zig"); diff --git a/src/unicode/grapheme.zig b/src/unicode/grapheme.zig index 2ba36161f..66ba27b12 100644 --- a/src/unicode/grapheme.zig +++ b/src/unicode/grapheme.zig @@ -149,49 +149,6 @@ fn graphemeBreakClass( return true; } -// This test will verify the grapheme break implementation. This iterates over billions of codepoints so it is SLOW. -// It's not meant to be run in CI, but it's useful for debugging. -test "grapheme break check against ziglyph" { - const ziglyph = @import("ziglyph"); - - // Set the min and max to control the test range. - const min = 0; - const max = std.math.maxInt(u21) + 1; - var success: bool = true; - - var state: BreakState = .{}; - var zg_state: u3 = 0; - for (min..max) |cp1| { - if (cp1 == '\r' or cp1 == '\n' or - ziglyph.grapheme_break.isControl(@intCast(cp1))) continue; - - for (min..max) |cp2| { - if (cp2 == '\r' or cp2 == '\n' or - ziglyph.grapheme_break.isControl(@intCast(cp2))) continue; - - const gb = graphemeBreak(@intCast(cp1), @intCast(cp2), &state); - const zg_gb = ziglyph.graphemeBreak(@intCast(cp1), @intCast(cp2), &zg_state); - if (gb != zg_gb) { - success = false; - std.log.warn("cp1={x} cp2={x} gb={} state={} zg_gb={} zg_state={}", .{ - cp1, - cp2, - gb, - state, - zg_gb, - zg_state, - }); - } - } - } - - try std.testing.expect(success); -} - -pub const std_options = struct { - pub const log_level: std.log.Level = .info; -}; - test "grapheme break: emoji modifier" { const testing = std.testing; diff --git a/src/unicode/main.zig b/src/unicode/main.zig index e8ba05b72..8d297569a 100644 --- a/src/unicode/main.zig +++ b/src/unicode/main.zig @@ -1,3 +1,4 @@ +const std = @import("std"); pub const lut = @import("lut.zig"); const grapheme = @import("grapheme.zig"); @@ -10,3 +11,113 @@ pub const GraphemeBreakState = grapheme.BreakState; test { @import("std").testing.refAllDecls(@This()); } + +/// Build Ghostty with `zig build -Doptimize=ReleaseFast -Demit-unicode-test`. +/// +/// Usage: ./zig-out/bin/unicode-test [grapheme|width|all] [zg|ziglyph|all] +/// +/// grapheme: this will verify the grapheme break implementation. This +/// iterates over billions of codepoints so it is SLOW. +/// +/// width: this verifies the table codepoint widths match +/// zg: compare grapheme/width against zg +/// ziglyph: compare grapheme/width against ziglyph +pub fn main() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + + const alloc = gpa.allocator(); + + const args = try std.process.argsAlloc(alloc); + defer std.process.argsFree(alloc, args); + + var zg = try props.init(alloc); + defer zg.deinit(alloc); + + const ziglyph = @import("ziglyph"); + const Graphemes = @import("Graphemes"); + const DisplayWidth = @import("DisplayWidth"); + + const testAll = args.len < 2 or std.mem.eql(u8, args[1], "all"); + const compareAll = args.len < 3 or std.mem.eql(u8, args[2], "all"); + const compareZg = compareAll or std.mem.eql(u8, args[2], "zg"); + const compareZiglyph = compareAll or std.mem.eql(u8, args[2], "ziglyph"); + + // Set the min and max to control the test range. + const min = 0; + const max = 0x110000; + + var state: GraphemeBreakState = .{}; + var zg_state: Graphemes.State = .{}; + var ziglyph_state: u3 = 0; + + if (testAll or std.mem.eql(u8, args[1], "grapheme")) { + std.log.info("============== testing grapheme break ===============", .{}); + + for (min..max) |cp1| { + if (cp1 % 0x100 == 0) std.log.info("progress: cp1={x}", .{cp1}); + + if (cp1 == '\r' or cp1 == '\n' or + Graphemes.gbp(zg.graphemes, @intCast(cp1)) == .Control) continue; + + for (min..max) |cp2| { + if (cp2 == '\r' or cp2 == '\n' or + Graphemes.gbp(zg.graphemes, @intCast(cp1)) == .Control) continue; + + const gb = graphemeBreak(@intCast(cp1), @intCast(cp2), &state); + if (compareZg) { + const zg_gb = Graphemes.graphemeBreak(@intCast(cp1), @intCast(cp2), &zg.graphemes, &zg_state); + if (gb != zg_gb) { + std.log.warn("[zg mismatch] cp1={x} cp2={x} gb={} zg_gb={} state={} zg_state={}", .{ + cp1, + cp2, + gb, + zg_gb, + state, + zg_state, + }); + } + } + if (compareZiglyph) { + const ziglyph_gb = ziglyph.graphemeBreak(@intCast(cp1), @intCast(cp2), &ziglyph_state); + if (gb != ziglyph_gb) { + std.log.warn("[ziglyph mismatch] cp1={x} cp2={x} gb={} ziglyph_gb={} state={} ziglyph_state={}", .{ + cp1, + cp2, + gb, + ziglyph_gb, + state, + ziglyph_state, + }); + } + } + } + } + } + + if (testAll or std.mem.eql(u8, args[1], "width")) { + std.log.info("============== testing codepoint width ==============", .{}); + + for (min..max) |cp| { + if (cp % 0x10000 == 0) std.log.info("progress: cp={x}", .{cp}); + + const t = table.get(@intCast(cp)); + if (compareZg) { + const zg_width = @min(2, @max(0, DisplayWidth.codePointWidth(zg.display_width, @intCast(cp)))); + if (t.width != zg_width) { + std.log.warn("[zg mismatch] cp={x} t={} zg={}", .{ cp, t.width, zg_width }); + } + } + if (compareZiglyph) { + const ziglyph_width = @min(2, @max(0, DisplayWidth.codePointWidth(zg.display_width, @intCast(cp)))); + if (t.width != ziglyph_width) { + std.log.warn("[ziglyph mismatch] cp={x} t={} zg={}", .{ cp, t.width, ziglyph_width }); + } + } + } + } +} + +pub const std_options: std.Options = .{ + .log_level = .debug, +}; diff --git a/src/unicode/props.zig b/src/unicode/props.zig index c4c147c70..86553a8bf 100644 --- a/src/unicode/props.zig +++ b/src/unicode/props.zig @@ -8,7 +8,8 @@ const lut = @import("lut.zig"); graphemes: Graphemes, display_width: DisplayWidth, -fn init(alloc: std.mem.Allocator) !props { +// Public only for unicode-test +pub fn init(alloc: std.mem.Allocator) !props { const graphemes = try Graphemes.init(alloc); return .{ .graphemes = graphemes, @@ -16,7 +17,8 @@ fn init(alloc: std.mem.Allocator) !props { }; } -fn deinit(self: *props, alloc: std.mem.Allocator) void { +// Public only for unicode-test +pub fn deinit(self: *props, alloc: std.mem.Allocator) void { self.graphemes.deinit(alloc); self.display_width.deinit(alloc); } @@ -180,22 +182,3 @@ pub fn main() !void { // t.stage3.len, // }); } - -// This is not very fast in debug modes, so its commented by default. -// IMPORTANT: UNCOMMENT THIS WHENEVER MAKING CODEPOINTWIDTH CHANGES. -//test "tables match zg" { -// const testing = std.testing; -// -// const display_width = try DisplayWidth.init(std.testing.allocator); -// defer display_width.deinit(std.testing.allocator); -// -// const min = 0xFF + 1; // start outside ascii -// for (min..0x110000) |cp| { -// const t = table.get(@intCast(cp)); -// const zg = @min(2, @max(0, DisplayWidth.codePointWidth(display_width, @intCast(cp)))); -// if (t.width != zg) { -// std.log.warn("mismatch cp=U+{x} t={} zg={}", .{ cp, t, zg }); -// try testing.expect(false); -// } -// } -//} From c0d818aac5f1bd7b25dedb324f7a9f5f3cb8c5bb Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Fri, 4 Jul 2025 12:47:48 -0400 Subject: [PATCH 03/12] Update zg --- build.zig.zon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index e1bf7c814..22360b102 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -42,8 +42,8 @@ .lazy = true, }, .zg = .{ - .url = "https://codeberg.org/jacobsandlund/zg/archive/21be4c72f2aff537d7630a7aeb641befc935db89.tar.gz", - .hash = "zg-0.14.0-oGqU3IkvswJcetRSabEnsUe_jsWz48WkZebbVS-pJ5jB", + .url = "https://codeberg.org/jacobsandlund/zg/archive/1ff1fc206db017444e80db4ccfc24378c16735e0.tar.gz", + .hash = "zg-0.14.0-oGqU3DAwswKPRCEJKlJpPKA8mlMb_1iX6vFqK-c6BenW", .lazy = true, }, .zig_wayland = .{ From 025895dc905505517bcde56bf0e58a44d54c0428 Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Fri, 4 Jul 2025 14:41:50 -0400 Subject: [PATCH 04/12] comparing to old version of table (ziglyph based) --- src/build/GhosttyUnicodeTest.zig | 11 ++ src/build/UnicodeTables.zig | 12 ++ src/unicode/grapheme.zig | 14 +++ src/unicode/main.zig | 210 +++++++++++++++++++++++-------- src/unicode/props.zig | 56 ++++++++- 5 files changed, 250 insertions(+), 53 deletions(-) diff --git a/src/build/GhosttyUnicodeTest.zig b/src/build/GhosttyUnicodeTest.zig index db3575c79..8421e9fae 100644 --- a/src/build/GhosttyUnicodeTest.zig +++ b/src/build/GhosttyUnicodeTest.zig @@ -3,6 +3,7 @@ const UnicodeTest = @This(); const std = @import("std"); const Config = @import("Config.zig"); const SharedDeps = @import("SharedDeps.zig"); +const UnicodeTables = @import("UnicodeTables.zig"); /// The unicode test executable. exe: *std.Build.Step.Compile, @@ -27,6 +28,7 @@ pub fn init(b: *std.Build, cfg: *const Config, deps: *const SharedDeps) !Unicode // Add the shared dependencies _ = try deps.add(exe); + // Add ziglyph just for unicode-test if (b.lazyDependency("ziglyph", .{ .target = cfg.target, .optimize = cfg.optimize, @@ -34,6 +36,15 @@ pub fn init(b: *std.Build, cfg: *const Config, deps: *const SharedDeps) !Unicode exe.root_module.addImport("ziglyph", dep.module("ziglyph")); } + // Add the old version of the unicode tables + const old_unicode_tables = try UnicodeTables.init(b); + old_unicode_tables.run.addArg("old"); + + old_unicode_tables.output.addStepDependencies(&exe.step); + exe.root_module.addAnonymousImport("old_unicode_tables", .{ + .root_source_file = old_unicode_tables.output, + }); + return .{ .exe = exe, .install_step = install_step, diff --git a/src/build/UnicodeTables.zig b/src/build/UnicodeTables.zig index 10f2064f7..ef2fc8cd9 100644 --- a/src/build/UnicodeTables.zig +++ b/src/build/UnicodeTables.zig @@ -6,6 +6,9 @@ const Config = @import("Config.zig"); /// The exe. exe: *std.Build.Step.Compile, +/// The run artifact for the exe. +run: *std.Build.Step.Run, + /// The output path for the unicode tables output: std.Build.LazyPath, @@ -28,9 +31,18 @@ pub fn init(b: *std.Build) !UnicodeTables { exe.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); } + // Only used if we're building the old unicode tables + if (b.lazyDependency("ziglyph", .{ + .target = b.graph.host, + })) |dep| { + exe.root_module.addImport("ziglyph", dep.module("ziglyph")); + } + const run = b.addRunArtifact(exe); + return .{ .exe = exe, + .run = run, .output = run.captureStdOut(), }; } diff --git a/src/unicode/grapheme.zig b/src/unicode/grapheme.zig index 66ba27b12..e88c9820e 100644 --- a/src/unicode/grapheme.zig +++ b/src/unicode/grapheme.zig @@ -2,6 +2,7 @@ const std = @import("std"); const props = @import("props.zig"); const GraphemeBoundaryClass = props.GraphemeBoundaryClass; const table = props.table; +const oldTable = props.oldTable; /// Determines if there is a grapheme break between two codepoints. This /// must be called sequentially maintaining the state between calls. @@ -22,6 +23,19 @@ pub fn graphemeBreak(cp1: u21, cp2: u21, state: *BreakState) bool { return value.result; } +/// Only used for unicode-test. +pub fn oldGraphemeBreak(cp1: u21, cp2: u21, state: *BreakState) bool { + const value = Precompute.data[ + (Precompute.Key{ + .gbc1 = oldTable.get(cp1).grapheme_boundary_class, + .gbc2 = oldTable.get(cp2).grapheme_boundary_class, + .state = state.*, + }).index() + ]; + state.* = value.state; + return value.result; +} + /// The state that must be maintained between calls to `graphemeBreak`. pub const BreakState = packed struct(u2) { extended_pictographic: bool = false, diff --git a/src/unicode/main.zig b/src/unicode/main.zig index 8d297569a..e31f3f623 100644 --- a/src/unicode/main.zig +++ b/src/unicode/main.zig @@ -14,14 +14,24 @@ test { /// Build Ghostty with `zig build -Doptimize=ReleaseFast -Demit-unicode-test`. /// -/// Usage: ./zig-out/bin/unicode-test [grapheme|width|all] [zg|ziglyph|all] -/// -/// grapheme: this will verify the grapheme break implementation. This -/// iterates over billions of codepoints so it is SLOW. +/// Usage: ./zig-out/bin/unicode-test [width|class|break|all] [zg|ziglyph|old|all] /// /// width: this verifies the table codepoint widths match +/// class: this verifies the table grapheme boundary classes match +/// break: this will verify the grapheme break implementation. This +/// iterates over billions of codepoints so it is SLOW. +/// /// zg: compare grapheme/width against zg /// ziglyph: compare grapheme/width against ziglyph +/// old: compare grapheme against old implementation +/// +/// Note: To enable `old` comparisons, uncomment sections of these files +/// (search for "old"): +/// * ./main.zig (this file) +/// * ./props.zig +/// * ./grapheme.zig +/// * src/build/GhosttyUnicodeTest.zig +/// * src/build/UnicodeTables.zig pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); @@ -42,59 +52,12 @@ pub fn main() !void { const compareAll = args.len < 3 or std.mem.eql(u8, args[2], "all"); const compareZg = compareAll or std.mem.eql(u8, args[2], "zg"); const compareZiglyph = compareAll or std.mem.eql(u8, args[2], "ziglyph"); + const compareOld = compareAll or std.mem.eql(u8, args[2], "old"); // Set the min and max to control the test range. const min = 0; const max = 0x110000; - var state: GraphemeBreakState = .{}; - var zg_state: Graphemes.State = .{}; - var ziglyph_state: u3 = 0; - - if (testAll or std.mem.eql(u8, args[1], "grapheme")) { - std.log.info("============== testing grapheme break ===============", .{}); - - for (min..max) |cp1| { - if (cp1 % 0x100 == 0) std.log.info("progress: cp1={x}", .{cp1}); - - if (cp1 == '\r' or cp1 == '\n' or - Graphemes.gbp(zg.graphemes, @intCast(cp1)) == .Control) continue; - - for (min..max) |cp2| { - if (cp2 == '\r' or cp2 == '\n' or - Graphemes.gbp(zg.graphemes, @intCast(cp1)) == .Control) continue; - - const gb = graphemeBreak(@intCast(cp1), @intCast(cp2), &state); - if (compareZg) { - const zg_gb = Graphemes.graphemeBreak(@intCast(cp1), @intCast(cp2), &zg.graphemes, &zg_state); - if (gb != zg_gb) { - std.log.warn("[zg mismatch] cp1={x} cp2={x} gb={} zg_gb={} state={} zg_state={}", .{ - cp1, - cp2, - gb, - zg_gb, - state, - zg_state, - }); - } - } - if (compareZiglyph) { - const ziglyph_gb = ziglyph.graphemeBreak(@intCast(cp1), @intCast(cp2), &ziglyph_state); - if (gb != ziglyph_gb) { - std.log.warn("[ziglyph mismatch] cp1={x} cp2={x} gb={} ziglyph_gb={} state={} ziglyph_state={}", .{ - cp1, - cp2, - gb, - ziglyph_gb, - state, - ziglyph_state, - }); - } - } - } - } - } - if (testAll or std.mem.eql(u8, args[1], "width")) { std.log.info("============== testing codepoint width ==============", .{}); @@ -116,6 +79,149 @@ pub fn main() !void { } } } + + if (testAll or std.mem.eql(u8, args[1], "class")) { + std.log.info("============== testing grapheme boundary class ======", .{}); + + for (min..max) |cp| { + if (cp % 0x10000 == 0) std.log.info("progress: cp={x}", .{cp}); + + const t = table.get(@intCast(cp)); + + if (compareZg) { + const gbp = Graphemes.gbp(zg.graphemes, @intCast(cp)); + const matches = switch (t.grapheme_boundary_class) { + .extended_pictographic_base => gbp == .Emoji_Modifier_Base, + .emoji_modifier => gbp == .Emoji_Modifier, + .extended_pictographic => gbp == .Extended_Pictographic, + .L => gbp == .L, + .V => gbp == .V, + .T => gbp == .T, + .LV => gbp == .LV, + .LVT => gbp == .LVT, + .prepend => gbp == .Prepend, + .extend => gbp == .Extend, + .zwj => gbp == .ZWJ, + .spacing_mark => gbp == .SpacingMark, + .regional_indicator => gbp == .Regional_Indicator, + .invalid => gbp == .none or gbp == .Control or gbp == .CR or gbp == .LF, + }; + + if (!matches) { + std.log.warn("[zg mismatch] cp={x} t={} zg={}", .{ cp, t.grapheme_boundary_class, gbp }); + } + } + + if (compareZiglyph) { + const ziglyph_valid = (ziglyph.emoji.isEmojiModifierBase(@intCast(cp)) or + ziglyph.emoji.isEmojiModifier(@intCast(cp)) or + ziglyph.emoji.isExtendedPictographic(@intCast(cp)) or + ziglyph.grapheme_break.isL(@intCast(cp)) or + ziglyph.grapheme_break.isV(@intCast(cp)) or + ziglyph.grapheme_break.isT(@intCast(cp)) or + ziglyph.grapheme_break.isLv(@intCast(cp)) or + ziglyph.grapheme_break.isLvt(@intCast(cp)) or + ziglyph.grapheme_break.isPrepend(@intCast(cp)) or + ziglyph.grapheme_break.isExtend(@intCast(cp)) or + ziglyph.grapheme_break.isZwj(@intCast(cp)) or + ziglyph.grapheme_break.isSpacingmark(@intCast(cp)) or + ziglyph.grapheme_break.isRegionalIndicator(@intCast(cp))); + + const matches = switch (t.grapheme_boundary_class) { + .extended_pictographic_base => ziglyph.emoji.isEmojiModifierBase(@intCast(cp)), + .emoji_modifier => ziglyph.emoji.isEmojiModifier(@intCast(cp)), + .extended_pictographic => ziglyph.emoji.isExtendedPictographic(@intCast(cp)), + .L => ziglyph.grapheme_break.isL(@intCast(cp)), + .V => ziglyph.grapheme_break.isV(@intCast(cp)), + .T => ziglyph.grapheme_break.isT(@intCast(cp)), + .LV => ziglyph.grapheme_break.isLv(@intCast(cp)), + .LVT => ziglyph.grapheme_break.isLvt(@intCast(cp)), + .prepend => ziglyph.grapheme_break.isPrepend(@intCast(cp)), + .extend => ziglyph.grapheme_break.isExtend(@intCast(cp)), + .zwj => ziglyph.grapheme_break.isZwj(@intCast(cp)), + .spacing_mark => ziglyph.grapheme_break.isSpacingmark(@intCast(cp)), + .regional_indicator => ziglyph.grapheme_break.isRegionalIndicator(@intCast(cp)), + .invalid => !ziglyph_valid, + }; + + if (!matches) { + std.log.warn("[ziglyph mismatch] cp={x} t={} ziglyph_valid={}", .{ cp, t.grapheme_boundary_class, ziglyph_valid }); + } + } + + if (compareOld) { + const oldT = props.oldTable.get(@intCast(cp)); + if (oldT.grapheme_boundary_class != t.grapheme_boundary_class) { + std.log.warn("[old mismatch] cp={x} t={} old={}", .{ cp, t.grapheme_boundary_class, oldT.grapheme_boundary_class }); + } + } + } + } + + var state: GraphemeBreakState = .{}; + var old_state: GraphemeBreakState = .{}; + var zg_state: Graphemes.State = .{}; + var ziglyph_state: u3 = 0; + + if (testAll or std.mem.eql(u8, args[1], "break")) { + std.log.info("============== testing grapheme break ===============", .{}); + + for (min..max) |cp1| { + if (cp1 % 0x100 == 0) std.log.info("progress: cp1={x}", .{cp1}); + + if (cp1 == '\r' or cp1 == '\n' or + Graphemes.gbp(zg.graphemes, @intCast(cp1)) == .Control) continue; + + for (min..max) |cp2| { + if (cp2 == '\r' or cp2 == '\n' or + Graphemes.gbp(zg.graphemes, @intCast(cp1)) == .Control) continue; + + const gb = graphemeBreak(@intCast(cp1), @intCast(cp2), &state); + + if (compareOld) { + const old_gb = grapheme.oldGraphemeBreak(@intCast(cp1), @intCast(cp2), &old_state); + if (gb != old_gb) { + std.log.warn("[old mismatch] cp1={x} cp2={x} gb={} old_gb={} state={} old_state={}", .{ + cp1, + cp2, + gb, + old_gb, + state, + old_state, + }); + } + } + + if (compareZg) { + const zg_gb = Graphemes.graphemeBreak(@intCast(cp1), @intCast(cp2), &zg.graphemes, &zg_state); + if (gb != zg_gb) { + std.log.warn("[zg mismatch] cp1={x} cp2={x} gb={} zg_gb={} state={} zg_state={}", .{ + cp1, + cp2, + gb, + zg_gb, + state, + zg_state, + }); + } + } + + if (compareZiglyph) { + const ziglyph_gb = ziglyph.graphemeBreak(@intCast(cp1), @intCast(cp2), &ziglyph_state); + if (gb != ziglyph_gb) { + std.log.warn("[ziglyph mismatch] cp1={x} cp2={x} gb={} ziglyph_gb={} state={} ziglyph_state={}", .{ + cp1, + cp2, + gb, + ziglyph_gb, + state, + ziglyph_state, + }); + } + } + } + } + } } pub const std_options: std.Options = .{ diff --git a/src/unicode/props.zig b/src/unicode/props.zig index 86553a8bf..3b8364235 100644 --- a/src/unicode/props.zig +++ b/src/unicode/props.zig @@ -8,6 +8,9 @@ const lut = @import("lut.zig"); graphemes: Graphemes, display_width: DisplayWidth, +// Whether to use the old implementation based on ziglyph. +old: bool = false, + // Public only for unicode-test pub fn init(alloc: std.mem.Allocator) !props { const graphemes = try Graphemes.init(alloc); @@ -36,6 +39,19 @@ pub const table = table: { }; }; +/// The old lookup tables for Ghostty. Only used for unicode-test. +pub const oldTable = table: { + // This is only available after running main() below as part of the Ghostty + // build.zig, but due to Zig's lazy analysis we can still reference it here. + const generated = @import("old_unicode_tables").Tables(Properties); + const Tables = lut.Tables(Properties); + break :table Tables{ + .stage1 = &generated.stage1, + .stage2 = &generated.stage2, + .stage3 = &generated.stage3, + }; +}; + /// Property set per codepoint that Ghostty cares about. /// /// Adding to this lets you find new properties but also potentially makes @@ -120,6 +136,36 @@ pub const GraphemeBoundaryClass = enum(u4) { }; } + pub fn initOld(cp: u21) GraphemeBoundaryClass { + const ziglyph = @import("ziglyph"); + + // We special-case modifier bases because we should not break + // if a modifier isn't next to a base. + if (ziglyph.emoji.isEmojiModifierBase(cp)) { + assert(ziglyph.emoji.isExtendedPictographic(cp)); + return .extended_pictographic_base; + } + + if (ziglyph.emoji.isEmojiModifier(cp)) return .emoji_modifier; + if (ziglyph.emoji.isExtendedPictographic(cp)) return .extended_pictographic; + if (ziglyph.grapheme_break.isL(cp)) return .L; + if (ziglyph.grapheme_break.isV(cp)) return .V; + if (ziglyph.grapheme_break.isT(cp)) return .T; + if (ziglyph.grapheme_break.isLv(cp)) return .LV; + if (ziglyph.grapheme_break.isLvt(cp)) return .LVT; + if (ziglyph.grapheme_break.isPrepend(cp)) return .prepend; + if (ziglyph.grapheme_break.isExtend(cp)) return .extend; + if (ziglyph.grapheme_break.isZwj(cp)) return .zwj; + if (ziglyph.grapheme_break.isSpacingmark(cp)) return .spacing_mark; + if (ziglyph.grapheme_break.isRegionalIndicator(cp)) return .regional_indicator; + + // This is obviously not INVALID invalid, there is SOME grapheme + // boundary class for every codepoint. But we don't care about + // anything that doesn't fit into the above categories. + + return .invalid; + } + /// Returns true if this is an extended pictographic type. This /// should be used instead of comparing the enum value directly /// because we classify multiple. @@ -145,7 +191,8 @@ pub fn get(ctx: props, cp: u21) !Properties { return .{ .width = @intCast(@min(2, @max(0, zg_width))), - .grapheme_boundary_class = .init(ctx, cp), + //.grapheme_boundary_class = .init(ctx, cp), + .grapheme_boundary_class = if (ctx.old) .initOld(cp) else .init(ctx, cp), }; } } @@ -161,9 +208,16 @@ pub fn main() !void { defer arena_state.deinit(); const alloc = arena_state.allocator(); + const args = try std.process.argsAlloc(alloc); + defer std.process.argsFree(alloc, args); + var self = try init(alloc); defer self.deinit(alloc); + if (args.len > 1 and std.mem.eql(u8, args[1], "old")) { + self.old = true; + } + const gen: lut.Generator( Properties, props, From 8a218149dd94f63557edea700baf0ff750b0cfa2 Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Fri, 4 Jul 2025 14:50:37 -0400 Subject: [PATCH 05/12] also compare to old widths --- src/unicode/main.zig | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/unicode/main.zig b/src/unicode/main.zig index e31f3f623..a3868f3c8 100644 --- a/src/unicode/main.zig +++ b/src/unicode/main.zig @@ -14,19 +14,19 @@ test { /// Build Ghostty with `zig build -Doptimize=ReleaseFast -Demit-unicode-test`. /// -/// Usage: ./zig-out/bin/unicode-test [width|class|break|all] [zg|ziglyph|old|all] +/// Usage: ./zig-out/bin/unicode-test [width|class|break|all] [old|zg|ziglyph|all] /// /// width: this verifies the table codepoint widths match /// class: this verifies the table grapheme boundary classes match /// break: this will verify the grapheme break implementation. This /// iterates over billions of codepoints so it is SLOW. /// -/// zg: compare grapheme/width against zg -/// ziglyph: compare grapheme/width against ziglyph -/// old: compare grapheme against old implementation +/// old: compare against old implementation +/// zg: compare against zg +/// ziglyph: compare against ziglyph /// -/// Note: To enable `old` comparisons, uncomment sections of these files -/// (search for "old"): +/// Note: To disable/enable `old` comparisons, (un)comment sections of these +/// files (search for "old"): /// * ./main.zig (this file) /// * ./props.zig /// * ./grapheme.zig @@ -50,9 +50,9 @@ pub fn main() !void { const testAll = args.len < 2 or std.mem.eql(u8, args[1], "all"); const compareAll = args.len < 3 or std.mem.eql(u8, args[2], "all"); + const compareOld = compareAll or std.mem.eql(u8, args[2], "old"); const compareZg = compareAll or std.mem.eql(u8, args[2], "zg"); const compareZiglyph = compareAll or std.mem.eql(u8, args[2], "ziglyph"); - const compareOld = compareAll or std.mem.eql(u8, args[2], "old"); // Set the min and max to control the test range. const min = 0; @@ -65,12 +65,21 @@ pub fn main() !void { if (cp % 0x10000 == 0) std.log.info("progress: cp={x}", .{cp}); const t = table.get(@intCast(cp)); + + if (compareOld) { + const oldT = props.oldTable.get(@intCast(cp)); + if (oldT.width != t.width) { + std.log.warn("[old mismatch] cp={x} t={} old={}", .{ cp, t.width, oldT.width }); + } + } + if (compareZg) { const zg_width = @min(2, @max(0, DisplayWidth.codePointWidth(zg.display_width, @intCast(cp)))); if (t.width != zg_width) { std.log.warn("[zg mismatch] cp={x} t={} zg={}", .{ cp, t.width, zg_width }); } } + if (compareZiglyph) { const ziglyph_width = @min(2, @max(0, DisplayWidth.codePointWidth(zg.display_width, @intCast(cp)))); if (t.width != ziglyph_width) { @@ -88,6 +97,13 @@ pub fn main() !void { const t = table.get(@intCast(cp)); + if (compareOld) { + const oldT = props.oldTable.get(@intCast(cp)); + if (oldT.grapheme_boundary_class != t.grapheme_boundary_class) { + std.log.warn("[old mismatch] cp={x} t={} old={}", .{ cp, t.grapheme_boundary_class, oldT.grapheme_boundary_class }); + } + } + if (compareZg) { const gbp = Graphemes.gbp(zg.graphemes, @intCast(cp)); const matches = switch (t.grapheme_boundary_class) { @@ -148,13 +164,6 @@ pub fn main() !void { std.log.warn("[ziglyph mismatch] cp={x} t={} ziglyph_valid={}", .{ cp, t.grapheme_boundary_class, ziglyph_valid }); } } - - if (compareOld) { - const oldT = props.oldTable.get(@intCast(cp)); - if (oldT.grapheme_boundary_class != t.grapheme_boundary_class) { - std.log.warn("[old mismatch] cp={x} t={} old={}", .{ cp, t.grapheme_boundary_class, oldT.grapheme_boundary_class }); - } - } } } From ca17f34d941d6697b1322802a2f2b1516e13c3b2 Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Fri, 4 Jul 2025 15:23:34 -0400 Subject: [PATCH 06/12] use Context instead of props. only import DisplayWidth when needed --- src/build/SharedDeps.zig | 5 +- src/unicode/main.zig | 2 +- src/unicode/props.zig | 99 +++++++++++++++++++++------------------- 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/src/build/SharedDeps.zig b/src/build/SharedDeps.zig index 909f727d3..ce70e0a03 100644 --- a/src/build/SharedDeps.zig +++ b/src/build/SharedDeps.zig @@ -416,8 +416,9 @@ pub fn add( .optimize = optimize, })) |dep| { step.root_module.addImport("CaseFolding", dep.module("CaseFolding")); - // Only needed for table gen, but still include to have build succeeed - step.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); + if (self.config.emit_unicode_test) { + step.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); + } step.root_module.addImport("Emoji", dep.module("Emoji")); step.root_module.addImport("GeneralCategories", dep.module("GeneralCategories")); step.root_module.addImport("Graphemes", dep.module("Graphemes")); diff --git a/src/unicode/main.zig b/src/unicode/main.zig index a3868f3c8..03544e310 100644 --- a/src/unicode/main.zig +++ b/src/unicode/main.zig @@ -41,7 +41,7 @@ pub fn main() !void { const args = try std.process.argsAlloc(alloc); defer std.process.argsFree(alloc, args); - var zg = try props.init(alloc); + var zg = try props.Context.init(alloc); defer zg.deinit(alloc); const ziglyph = @import("ziglyph"); diff --git a/src/unicode/props.zig b/src/unicode/props.zig index 3b8364235..bfdcb3533 100644 --- a/src/unicode/props.zig +++ b/src/unicode/props.zig @@ -1,30 +1,55 @@ -const props = @This(); const std = @import("std"); const assert = std.debug.assert; -const Graphemes = @import("Graphemes"); -const DisplayWidth = @import("DisplayWidth"); const lut = @import("lut.zig"); +const Graphemes = @import("Graphemes"); -graphemes: Graphemes, -display_width: DisplayWidth, +/// The context needed for lut generation. +pub const Context = struct { + graphemes: Graphemes, + display_width: DisplayWidth, -// Whether to use the old implementation based on ziglyph. -old: bool = false, + // Whether to use the old implementation based on ziglyph. + old: bool = false, -// Public only for unicode-test -pub fn init(alloc: std.mem.Allocator) !props { - const graphemes = try Graphemes.init(alloc); - return .{ - .graphemes = graphemes, - .display_width = try DisplayWidth.initWithGraphemes(alloc, graphemes), - }; -} + const DisplayWidth = @import("DisplayWidth"); -// Public only for unicode-test -pub fn deinit(self: *props, alloc: std.mem.Allocator) void { - self.graphemes.deinit(alloc); - self.display_width.deinit(alloc); -} + // Public only for unicode-test + pub fn init(alloc: std.mem.Allocator) !Context { + const graphemes = try Graphemes.init(alloc); + return .{ + .graphemes = graphemes, + .display_width = try DisplayWidth.initWithGraphemes(alloc, graphemes), + }; + } + + // Public only for unicode-test + pub fn deinit(self: *Context, alloc: std.mem.Allocator) void { + self.graphemes.deinit(alloc); + self.display_width.deinit(alloc); + } + + pub fn get(self: Context, cp: u21) !Properties { + if (cp > 0x10FFFF) { + return .{ + .width = 0, + .grapheme_boundary_class = .invalid, + }; + } else { + const zg_width = DisplayWidth.codePointWidth(self.display_width, cp); + + return .{ + .width = @intCast(@min(2, @max(0, zg_width))), + //.grapheme_boundary_class = .init(self, cp), + .grapheme_boundary_class = if (self.old) .initOld(cp) else .init(self, cp), + }; + } + } + + pub fn eql(self: Context, a: Properties, b: Properties) bool { + _ = self; + return a.eql(b); + } +}; /// The lookup tables for Ghostty. pub const table = table: { @@ -114,7 +139,7 @@ pub const GraphemeBoundaryClass = enum(u4) { /// Gets the grapheme boundary class for a codepoint. This is VERY /// SLOW. The use case for this is only in generating lookup tables. - pub fn init(ctx: props, cp: u21) GraphemeBoundaryClass { + pub fn init(ctx: Context, cp: u21) GraphemeBoundaryClass { return switch (Graphemes.gbp(ctx.graphemes, cp)) { .Emoji_Modifier_Base => .extended_pictographic_base, .Emoji_Modifier => .emoji_modifier, @@ -180,28 +205,6 @@ pub const GraphemeBoundaryClass = enum(u4) { } }; -pub fn get(ctx: props, cp: u21) !Properties { - if (cp > 0x10FFFF) { - return .{ - .width = 0, - .grapheme_boundary_class = .invalid, - }; - } else { - const zg_width = DisplayWidth.codePointWidth(ctx.display_width, cp); - - return .{ - .width = @intCast(@min(2, @max(0, zg_width))), - //.grapheme_boundary_class = .init(ctx, cp), - .grapheme_boundary_class = if (ctx.old) .initOld(cp) else .init(ctx, cp), - }; - } -} - -pub fn eql(ctx: props, a: Properties, b: Properties) bool { - _ = ctx; - return a.eql(b); -} - /// Runnable binary to generate the lookup tables and output to stdout. pub fn main() !void { var arena_state = std.heap.ArenaAllocator.init(std.heap.page_allocator); @@ -211,17 +214,17 @@ pub fn main() !void { const args = try std.process.argsAlloc(alloc); defer std.process.argsFree(alloc, args); - var self = try init(alloc); - defer self.deinit(alloc); + var ctx = try Context.init(alloc); + defer ctx.deinit(alloc); if (args.len > 1 and std.mem.eql(u8, args[1], "old")) { - self.old = true; + ctx.old = true; } const gen: lut.Generator( Properties, - props, - ) = .{ .ctx = self }; + Context, + ) = .{ .ctx = ctx }; const t = try gen.generate(alloc); defer alloc.free(t.stage1); From a83e29c5957b4bea22909ed441b57d246be78b1e Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Sat, 5 Jul 2025 10:07:42 -0400 Subject: [PATCH 07/12] remove dependency on Graphemes outside of unicode test/table --- src/build/GhosttyUnicodeTest.zig | 9 +++++++++ src/build/SharedDeps.zig | 4 ---- src/crash/sentry.zig | 2 +- src/font/shaper/web_canvas.zig | 7 +++---- src/global.zig | 4 ---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/build/GhosttyUnicodeTest.zig b/src/build/GhosttyUnicodeTest.zig index 8421e9fae..4063c40ad 100644 --- a/src/build/GhosttyUnicodeTest.zig +++ b/src/build/GhosttyUnicodeTest.zig @@ -36,6 +36,15 @@ pub fn init(b: *std.Build, cfg: *const Config, deps: *const SharedDeps) !Unicode exe.root_module.addImport("ziglyph", dep.module("ziglyph")); } + // Add zg dependencies used just for unicode-test + if (b.lazyDependency("zg", .{ + .target = cfg.target, + .optimize = cfg.optimize, + })) |dep| { + exe.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); + exe.root_module.addImport("Graphemes", dep.module("Graphemes")); + } + // Add the old version of the unicode tables const old_unicode_tables = try UnicodeTables.init(b); old_unicode_tables.run.addArg("old"); diff --git a/src/build/SharedDeps.zig b/src/build/SharedDeps.zig index ce70e0a03..9d98a7c9c 100644 --- a/src/build/SharedDeps.zig +++ b/src/build/SharedDeps.zig @@ -416,12 +416,8 @@ pub fn add( .optimize = optimize, })) |dep| { step.root_module.addImport("CaseFolding", dep.module("CaseFolding")); - if (self.config.emit_unicode_test) { - step.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); - } step.root_module.addImport("Emoji", dep.module("Emoji")); step.root_module.addImport("GeneralCategories", dep.module("GeneralCategories")); - step.root_module.addImport("Graphemes", dep.module("Graphemes")); step.root_module.addImport("LetterCasing", dep.module("LetterCasing")); } if (b.lazyDependency("zf", .{ diff --git a/src/crash/sentry.zig b/src/crash/sentry.zig index 820c3e9a1..1b3fd68d8 100644 --- a/src/crash/sentry.zig +++ b/src/crash/sentry.zig @@ -58,7 +58,7 @@ pub fn init(gpa: Allocator) !void { // defer { // const end = std.time.Instant.now() catch unreachable; // // "[updateFrame critical time] \t" - // std.log.err("[sentry init time] start={}us duration={}ns", .{ start_micro, end.since(start) / std.time.ns_per_us }); + // std.log.err("[sentry init time] start={}us duration={}us", .{ start_micro, end.since(start) / std.time.ns_per_us }); // } // Must only start once diff --git a/src/font/shaper/web_canvas.zig b/src/font/shaper/web_canvas.zig index f6b39c7cd..e06ff9a99 100644 --- a/src/font/shaper/web_canvas.zig +++ b/src/font/shaper/web_canvas.zig @@ -4,7 +4,7 @@ const Allocator = std.mem.Allocator; const Graphemes = @import("Graphemes"); const font = @import("../main.zig"); const terminal = @import("../../terminal/main.zig"); -const zg = &@import("../../global.zig").state.zg; +const unicode = @import("../../unicode/main.zig"); const log = std.log.scoped(.font_shaper); @@ -112,7 +112,7 @@ pub const Shaper = struct { // font ligatures. However, we do support grapheme clustering. // This means we can render things like skin tone emoji but // we can't render things like single glyph "=>". - var break_state: Graphemes.State = .{}; + var break_state: unicode.GraphemeBreakState = .{}; var cp1: u21 = @intCast(codepoints[0]); var start: usize = 0; @@ -127,10 +127,9 @@ pub const Shaper = struct { const cp2: u21 = @intCast(codepoints[i]); defer cp1 = cp2; - break :blk Graphemes.graphemeBreak( + break :blk unicode.graphemeBreak( cp1, cp2, - zg.graphemes, &break_state, ); }; diff --git a/src/global.zig b/src/global.zig index cdd3541dc..85dc89ba4 100644 --- a/src/global.zig +++ b/src/global.zig @@ -13,7 +13,6 @@ const apprt = @import("apprt.zig"); const CaseFolding = @import("CaseFolding"); const Emoji = @import("Emoji"); const GeneralCategories = @import("GeneralCategories"); -const Graphemes = @import("Graphemes"); const LetterCasing = @import("LetterCasing"); const unicode = @import("unicode/main.zig"); @@ -232,7 +231,6 @@ pub const Zg = struct { case_folding: CaseFolding, emoji: Emoji, general_categories: GeneralCategories, - graphemes: Graphemes, letter_casing: LetterCasing, pub fn init(alloc: std.mem.Allocator) !Zg { @@ -240,7 +238,6 @@ pub const Zg = struct { .case_folding = try CaseFolding.init(alloc), .emoji = try Emoji.init(alloc), .general_categories = try GeneralCategories.init(alloc), - .graphemes = try Graphemes.init(alloc), .letter_casing = try LetterCasing.init(alloc), }; } @@ -249,7 +246,6 @@ pub const Zg = struct { self.case_folding.deinit(alloc); self.emoji.deinit(alloc); self.general_categories.deinit(alloc); - self.graphemes.deinit(alloc); self.letter_casing.deinit(alloc); } From 5485320687135495fe9c5f74510868a5255acfbd Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Mon, 7 Jul 2025 00:33:58 -0400 Subject: [PATCH 08/12] partial changes. forgot about the stash --- build.zig.zon | 4 +-- src/bench/codepoint-width.zig | 9 ++--- src/bench/grapheme-break.zig | 9 ++--- src/build/main.zig | 2 ++ src/config/Config.zig | 65 +--------------------------------- src/config/c_get.zig | 10 ------ src/config/formatter.zig | 4 --- src/font/CodepointResolver.zig | 9 +---- src/font/SharedGrid.zig | 2 -- src/font/SharedGridSet.zig | 8 ----- src/font/shaper/coretext.zig | 42 ---------------------- src/font/shaper/web_canvas.zig | 1 - src/global.zig | 44 ----------------------- src/input/Binding.zig | 43 ++-------------------- src/renderer/cell.zig | 6 ++-- src/unicode/main.zig | 19 ++++------ src/unicode/props.zig | 33 +++++------------ 17 files changed, 29 insertions(+), 281 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 72f9aa436..7d9490ee4 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -42,8 +42,8 @@ .lazy = true, }, .zg = .{ - .url = "https://codeberg.org/jacobsandlund/zg/archive/1ff1fc206db017444e80db4ccfc24378c16735e0.tar.gz", - .hash = "zg-0.14.0-oGqU3DAwswKPRCEJKlJpPKA8mlMb_1iX6vFqK-c6BenW", + .url = "https://codeberg.org/jacobsandlund/zg/archive/bfb71988dbef3d11e54637fa7f99a84a2c7c9d1f.tar.gz", + .hash = "zg-0.14.0-oGqU3A-9sgKPomBsiqZsVKUayWJ2oIwolsb_6PwM7PnX", .lazy = true, }, .zig_wayland = .{ diff --git a/src/bench/codepoint-width.zig b/src/bench/codepoint-width.zig index 90faafeef..ac4a366fa 100644 --- a/src/bench/codepoint-width.zig +++ b/src/bench/codepoint-width.zig @@ -64,10 +64,6 @@ pub fn main() !void { // We want to use the c allocator because it is much faster than GPA. const alloc = std.heap.c_allocator; - // Initialize DisplayWidth for zg - const display_width = try DisplayWidth.init(alloc); - display_width.deinit(alloc); - // Parse our args var args: Args = .{}; defer args.deinit(); @@ -84,7 +80,7 @@ pub fn main() !void { switch (args.mode) { .noop => try benchNoop(reader, buf), .wcwidth => try benchWcwidth(reader, buf), - .zg => try benchZg(display_width, reader, buf), + .zg => try benchZg(reader, buf), .simd => try benchSimd(reader, buf), .table => try benchTable(reader, buf), } @@ -160,7 +156,6 @@ noinline fn benchTable( } noinline fn benchZg( - display_width: DisplayWidth, reader: anytype, buf: []u8, ) !void { @@ -175,7 +170,7 @@ noinline fn benchZg( const cp_, const consumed = d.next(c); assert(consumed); if (cp_) |cp| { - const width = DisplayWidth.codePointWidth(display_width, cp); + const width = DisplayWidth.codePointWidth(cp); // Write the width to the buffer to avoid it being compiled away buf[0] = @intCast(width); diff --git a/src/bench/grapheme-break.zig b/src/bench/grapheme-break.zig index 1e889b01e..1c00a3e22 100644 --- a/src/bench/grapheme-break.zig +++ b/src/bench/grapheme-break.zig @@ -56,10 +56,6 @@ pub fn main() !void { // We want to use the c allocator because it is much faster than GPA. const alloc = std.heap.c_allocator; - // Initialize Graphemes for zg - const graphemes = try Graphemes.init(alloc); - graphemes.deinit(alloc); - // Parse our args var args: Args = .{}; defer args.deinit(); @@ -75,7 +71,7 @@ pub fn main() !void { // Handle the modes that do not depend on terminal state first. switch (args.mode) { .noop => try benchNoop(reader, buf), - .zg => try benchZg(&graphemes, reader, buf), + .zg => try benchZg(reader, buf), .table => try benchTable(reader, buf), } } @@ -123,7 +119,6 @@ noinline fn benchTable( } noinline fn benchZg( - graphemes: *const Graphemes, reader: anytype, buf: []u8, ) !void { @@ -140,7 +135,7 @@ noinline fn benchZg( const cp_, const consumed = d.next(c); assert(consumed); if (cp_) |cp2| { - const v = Graphemes.graphemeBreak(cp1, @intCast(cp2), graphemes, &state); + const v = Graphemes.graphemeBreak(cp1, @intCast(cp2), &state); buf[0] = @intCast(@intFromBool(v)); cp1 = cp2; } diff --git a/src/build/main.zig b/src/build/main.zig index 3154d395f..db61df8b2 100644 --- a/src/build/main.zig +++ b/src/build/main.zig @@ -15,6 +15,8 @@ pub const GhosttyFrameData = @import("GhosttyFrameData.zig"); pub const GhosttyLib = @import("GhosttyLib.zig"); pub const GhosttyResources = @import("GhosttyResources.zig"); pub const GhosttyI18n = @import("GhosttyI18n.zig"); +pub const GhosttyUnicodeTest = @import("GhosttyUnicodeTest.zig"); +pub const GhosttyXcodebuild = @import("GhosttyXcodebuild.zig"); pub const GhosttyXCFramework = @import("GhosttyXCFramework.zig"); pub const GhosttyWebdata = @import("GhosttyWebdata.zig"); pub const HelpStrings = @import("HelpStrings.zig"); diff --git a/src/config/Config.zig b/src/config/Config.zig index bf5354674..a53986bc9 100644 --- a/src/config/Config.zig +++ b/src/config/Config.zig @@ -16,6 +16,7 @@ const build_config = @import("../build_config.zig"); const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; +const global_state = &@import("../global.zig").state; const fontpkg = @import("../font/main.zig"); const inputpkg = @import("../input.zig"); const terminal = @import("../terminal/main.zig"); @@ -5991,8 +5992,6 @@ pub const Keybinds = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var set: Keybinds = .{}; try set.parseCLI(alloc, "shift+a=copy_to_clipboard"); @@ -6007,8 +6006,6 @@ pub const Keybinds = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: Keybinds = .{}; try list.parseCLI(alloc, "shift+a=csi:hello"); @@ -6025,8 +6022,6 @@ pub const Keybinds = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: Keybinds = .{}; try list.parseCLI(alloc, "ctrl+z>1=goto_tab:1"); @@ -6051,8 +6046,6 @@ pub const Keybinds = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: Keybinds = .{}; try list.parseCLI(alloc, "ctrl+a>ctrl+b>n=new_window"); @@ -6264,8 +6257,6 @@ pub const RepeatableCodepointMap = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: Self = .{}; try list.parseCLI(alloc, "U+ABCD=Comic Sans"); @@ -6303,8 +6294,6 @@ pub const RepeatableCodepointMap = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: Self = .{}; try list.parseCLI(alloc, "U+ABCD=Comic Sans"); @@ -6320,8 +6309,6 @@ pub const RepeatableCodepointMap = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: Self = .{}; try list.parseCLI(alloc, "U+0001 - U+0005=Verdana"); @@ -6337,8 +6324,6 @@ pub const RepeatableCodepointMap = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: Self = .{}; try list.parseCLI(alloc, "U+0006-U+0009, U+ABCD=Courier"); @@ -6417,8 +6402,6 @@ pub const FontStyle = union(enum) { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var p: Self = .{ .default = {} }; try p.parseCLI(alloc, "default"); @@ -6439,8 +6422,6 @@ pub const FontStyle = union(enum) { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var p: Self = .{ .default = {} }; try p.parseCLI(alloc, "default"); @@ -6456,8 +6437,6 @@ pub const FontStyle = union(enum) { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var p: Self = .{ .default = {} }; try p.parseCLI(alloc, "false"); @@ -6473,8 +6452,6 @@ pub const FontStyle = union(enum) { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var p: Self = .{ .default = {} }; try p.parseCLI(alloc, "bold"); @@ -6652,8 +6629,6 @@ pub const RepeatableCommand = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: RepeatableCommand = .{}; try list.parseCLI(alloc, "title:Foo,action:ignore"); @@ -6707,8 +6682,6 @@ pub const RepeatableCommand = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: RepeatableCommand = .{}; try list.parseCLI(alloc, "title:Bobr, action:text:Bober"); @@ -6724,8 +6697,6 @@ pub const RepeatableCommand = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var list: RepeatableCommand = .{}; try list.parseCLI(alloc, "title:Bobr, action:text:kurwa"); @@ -7444,8 +7415,6 @@ pub const Theme = struct { var arena = ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); // Single { @@ -7863,8 +7832,6 @@ const TestIterator = struct { test "parse hook: invalid command" { const testing = std.testing; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(testing.allocator); defer cfg.deinit(); const alloc = cfg._arena.?.allocator(); @@ -7876,8 +7843,6 @@ test "parse hook: invalid command" { test "parse e: command only" { const testing = std.testing; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(testing.allocator); defer cfg.deinit(); const alloc = cfg._arena.?.allocator(); @@ -7893,8 +7858,6 @@ test "parse e: command only" { test "parse e: command and args" { const testing = std.testing; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(testing.allocator); defer cfg.deinit(); const alloc = cfg._arena.?.allocator(); @@ -7913,8 +7876,6 @@ test "parse e: command and args" { test "clone default" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var source = try Config.default(alloc); defer source.deinit(); @@ -7932,8 +7893,6 @@ test "clone default" { test "clone preserves conditional state" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var a = try Config.default(alloc); defer a.deinit(); @@ -7962,8 +7921,6 @@ test "clone can then change conditional state" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -8024,8 +7981,6 @@ test "clone can then change conditional state" { test "clone preserves conditional set" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -8045,8 +8000,6 @@ test "clone preserves conditional set" { test "changed" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var source = try Config.default(alloc); defer source.deinit(); @@ -8061,8 +8014,6 @@ test "changed" { test "changeConditionalState ignores irrelevant changes" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); { var cfg = try Config.default(alloc); @@ -8082,8 +8033,6 @@ test "changeConditionalState ignores irrelevant changes" { test "changeConditionalState applies relevant changes" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); { var cfg = try Config.default(alloc); @@ -8106,8 +8055,6 @@ test "theme loading" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -8144,8 +8091,6 @@ test "theme loading preserves conditional state" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -8176,8 +8121,6 @@ test "theme priority is lower than config" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -8212,8 +8155,6 @@ test "theme loading correct light/dark" { var arena = ArenaAllocator.init(alloc); defer arena.deinit(); const alloc_arena = arena.allocator(); - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); // Setup our test theme var td = try internal_os.TempDir.init(); @@ -8303,8 +8244,6 @@ test "theme loading correct light/dark" { test "theme specifying light/dark changes window-theme from auto" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); { var cfg = try Config.default(alloc); @@ -8323,8 +8262,6 @@ test "theme specifying light/dark changes window-theme from auto" { test "theme specifying light/dark sets theme usage in conditional state" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); { var cfg = try Config.default(alloc); diff --git a/src/config/c_get.zig b/src/config/c_get.zig index 6be62bbc6..f235f596a 100644 --- a/src/config/c_get.zig +++ b/src/config/c_get.zig @@ -121,8 +121,6 @@ fn fieldByKey(self: *const Config, comptime k: Key) Value(k) { test "c_get: u8" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); @@ -136,8 +134,6 @@ test "c_get: u8" { test "c_get: enum" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); @@ -153,8 +149,6 @@ test "c_get: enum" { test "c_get: color" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); @@ -170,8 +164,6 @@ test "c_get: color" { test "c_get: optional" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); @@ -195,8 +187,6 @@ test "c_get: optional" { test "c_get: background-blur" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var c = try Config.default(alloc); defer c.deinit(); diff --git a/src/config/formatter.zig b/src/config/formatter.zig index 89298d425..cabf80953 100644 --- a/src/config/formatter.zig +++ b/src/config/formatter.zig @@ -193,8 +193,6 @@ pub const FileFormatter = struct { test "format default config" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -214,8 +212,6 @@ test "format default config" { test "format default config changed" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); cfg.@"font-size" = 42; diff --git a/src/font/CodepointResolver.zig b/src/font/CodepointResolver.zig index 5c87bfdb2..0f3808992 100644 --- a/src/font/CodepointResolver.zig +++ b/src/font/CodepointResolver.zig @@ -15,7 +15,6 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const Emoji = @import("Emoji"); const font = @import("main.zig"); -const zg = &@import("../global.zig").state.zg; const Atlas = font.Atlas; const CodepointMap = font.CodepointMap; const Collection = font.Collection; @@ -151,7 +150,7 @@ pub fn getIndex( // we'll do this multiple times if we recurse, but this is a cached function // call higher up (GroupCache) so this should be rare. const p_mode: Collection.PresentationMode = if (p) |v| .{ .explicit = v } else .{ - .default = if (Emoji.isEmojiPresentation(zg.emoji, @intCast(cp))) + .default = if (Emoji.isEmojiPresentation(@intCast(cp))) .emoji else .text, @@ -380,8 +379,6 @@ test getIndex { const testFont = font.embedded.regular; const testEmoji = font.embedded.emoji; const testEmojiText = font.embedded.emoji_text; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var lib = try Library.init(alloc); defer lib.deinit(); @@ -460,8 +457,6 @@ test "getIndex disabled font style" { const testing = std.testing; const alloc = testing.allocator; const testFont = font.embedded.regular; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var atlas_grayscale = try font.Atlas.init(alloc, 512, .grayscale); defer atlas_grayscale.deinit(alloc); @@ -517,8 +512,6 @@ test "getIndex disabled font style" { test "getIndex box glyph" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var lib = try Library.init(alloc); defer lib.deinit(); diff --git a/src/font/SharedGrid.zig b/src/font/SharedGrid.zig index 3389e4814..3ccac7fa1 100644 --- a/src/font/SharedGrid.zig +++ b/src/font/SharedGrid.zig @@ -393,8 +393,6 @@ fn testGrid(mode: TestMode, alloc: Allocator, lib: Library) !SharedGrid { test getIndex { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); // const testEmoji = @import("test.zig").fontEmoji; var lib = try Library.init(alloc); diff --git a/src/font/SharedGridSet.zig b/src/font/SharedGridSet.zig index 6c0df5efb..b77b44f23 100644 --- a/src/font/SharedGridSet.zig +++ b/src/font/SharedGridSet.zig @@ -711,8 +711,6 @@ pub const Key = struct { test "Key" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -732,8 +730,6 @@ test "Key" { test "Key different font points" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -752,8 +748,6 @@ test "Key different font points" { test "Key different font DPI" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var cfg = try Config.default(alloc); defer cfg.deinit(); @@ -772,8 +766,6 @@ test "Key different font DPI" { test SharedGridSet { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var set = try SharedGridSet.init(alloc); defer set.deinit(); diff --git a/src/font/shaper/coretext.zig b/src/font/shaper/coretext.zig index 508f8a1d5..f4f01d105 100644 --- a/src/font/shaper/coretext.zig +++ b/src/font/shaper/coretext.zig @@ -576,8 +576,6 @@ pub const Shaper = struct { test "run iterator" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -658,8 +656,6 @@ test "run iterator" { test "run iterator: empty cells with background set" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -708,8 +704,6 @@ test "run iterator: empty cells with background set" { test "shape" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -743,8 +737,6 @@ test "shape" { test "shape nerd fonts" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .nerd_font); defer testdata.deinit(); @@ -778,8 +770,6 @@ test "shape nerd fonts" { test "shape inconsolata ligs" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -834,8 +824,6 @@ test "shape inconsolata ligs" { test "shape monaspace ligs" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .monaspace_neon); defer testdata.deinit(); @@ -868,8 +856,6 @@ test "shape monaspace ligs" { test "shape left-replaced lig in last run" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .geist_mono); defer testdata.deinit(); @@ -902,8 +888,6 @@ test "shape left-replaced lig in last run" { test "shape left-replaced lig in early run" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .geist_mono); defer testdata.deinit(); @@ -933,8 +917,6 @@ test "shape left-replaced lig in early run" { test "shape U+3C9 with JB Mono" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaperWithFont(alloc, .jetbrains_mono); defer testdata.deinit(); @@ -966,8 +948,6 @@ test "shape U+3C9 with JB Mono" { test "shape emoji width" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -997,8 +977,6 @@ test "shape emoji width" { test "shape emoji width long" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1046,8 +1024,6 @@ test "shape emoji width long" { test "shape variation selector VS15" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1081,8 +1057,6 @@ test "shape variation selector VS15" { test "shape variation selector VS16" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1116,8 +1090,6 @@ test "shape variation selector VS16" { test "shape with empty cells in between" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1149,8 +1121,6 @@ test "shape with empty cells in between" { test "shape Chinese characters" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1191,8 +1161,6 @@ test "shape Chinese characters" { test "shape box glyphs" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1230,8 +1198,6 @@ test "shape box glyphs" { test "shape selection boundary" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1355,8 +1321,6 @@ test "shape selection boundary" { test "shape cursor boundary" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1494,8 +1458,6 @@ test "shape cursor boundary" { test "shape cursor boundary and colored emoji" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1590,8 +1552,6 @@ test "shape cursor boundary and colored emoji" { test "shape cell attribute change" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); @@ -1720,8 +1680,6 @@ test "shape high plane sprite font codepoint" { const testing = std.testing; const alloc = testing.allocator; - const zg = try @import("../../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var testdata = try testShaper(alloc); defer testdata.deinit(); diff --git a/src/font/shaper/web_canvas.zig b/src/font/shaper/web_canvas.zig index e06ff9a99..e0f0e1a00 100644 --- a/src/font/shaper/web_canvas.zig +++ b/src/font/shaper/web_canvas.zig @@ -1,7 +1,6 @@ const std = @import("std"); const assert = std.debug.assert; const Allocator = std.mem.Allocator; -const Graphemes = @import("Graphemes"); const font = @import("../main.zig"); const terminal = @import("../../terminal/main.zig"); const unicode = @import("../../unicode/main.zig"); diff --git a/src/global.zig b/src/global.zig index 85dc89ba4..668d2faec 100644 --- a/src/global.zig +++ b/src/global.zig @@ -10,11 +10,6 @@ const oni = @import("oniguruma"); const crash = @import("crash/main.zig"); const renderer = @import("renderer.zig"); const apprt = @import("apprt.zig"); -const CaseFolding = @import("CaseFolding"); -const Emoji = @import("Emoji"); -const GeneralCategories = @import("GeneralCategories"); -const LetterCasing = @import("LetterCasing"); -const unicode = @import("unicode/main.zig"); /// We export the xev backend we want to use so that the rest of /// Ghostty can import this once and have access to the proper @@ -38,7 +33,6 @@ pub const GlobalState = struct { action: ?cli.Action, logging: Logging, rlimits: ResourceLimits = .{}, - zg: Zg, /// The app resources directory, equivalent to zig-out/share when we build /// from source. This is null if we can't detect it. @@ -70,7 +64,6 @@ pub const GlobalState = struct { .logging = .{ .stderr = {} }, .rlimits = .{}, .resources_dir = .{}, - .zg = undefined, }; errdefer self.deinit(); @@ -157,8 +150,6 @@ pub const GlobalState = struct { ); }; - self.zg = try Zg.init(self.alloc); - // const sentrylib = @import("sentry"); // if (sentrylib.captureEvent(sentrylib.Value.initMessageEvent( // .info, @@ -197,8 +188,6 @@ pub const GlobalState = struct { // Flush our crash logs crash.deinit(); - self.zg.deinit(self.alloc); - if (self.gpa) |*value| { // We want to ensure that we deinit the GPA because this is // the point at which it will output if there were safety violations. @@ -227,39 +216,6 @@ pub const GlobalState = struct { } }; -pub const Zg = struct { - case_folding: CaseFolding, - emoji: Emoji, - general_categories: GeneralCategories, - letter_casing: LetterCasing, - - pub fn init(alloc: std.mem.Allocator) !Zg { - return .{ - .case_folding = try CaseFolding.init(alloc), - .emoji = try Emoji.init(alloc), - .general_categories = try GeneralCategories.init(alloc), - .letter_casing = try LetterCasing.init(alloc), - }; - } - - pub fn deinit(self: *Zg, alloc: std.mem.Allocator) void { - self.case_folding.deinit(alloc); - self.emoji.deinit(alloc); - self.general_categories.deinit(alloc); - self.letter_casing.deinit(alloc); - } - - pub fn initForTesting() !*Zg { - const zg = &state.zg; - zg.* = try Zg.init(std.testing.allocator); - return zg; - } - - pub fn deinitForTesting(self: *Zg) void { - self.deinit(std.testing.allocator); - } -}; - /// Maintains the Unix resource limits that we set for our process. This /// can be used to restore the limits to their original values. pub const ResourceLimits = struct { diff --git a/src/input/Binding.zig b/src/input/Binding.zig index 6719b7cd9..cab99d3d8 100644 --- a/src/input/Binding.zig +++ b/src/input/Binding.zig @@ -9,7 +9,6 @@ const LetterCasing = @import("LetterCasing"); const CaseFolding = @import("CaseFolding"); const key = @import("key.zig"); const KeyEvent = key.KeyEvent; -const zg = &@import("../global.zig").state.zg; /// The trigger that needs to be performed to execute the action. trigger: Trigger, @@ -1566,7 +1565,7 @@ pub const Trigger = struct { fn foldedCodepoint(cp: u21) [3]u21 { // ASCII fast path if (('A' <= cp and cp <= 'Z') or ('a' <= cp and cp <= 'z')) { - return .{ LetterCasing.toLower(zg.letter_casing, cp), 0, 0 }; + return .{ LetterCasing.toLower(cp), 0, 0 }; } // Unicode slow path. Case folding can result in more codepoints. @@ -1574,7 +1573,7 @@ pub const Trigger = struct { // as-is which isn't correct but until we have a failing test // then I don't want to handle this. var buf: [3]u21 = .{ 0, 0, 0 }; - _ = CaseFolding.caseFold(&zg.case_folding, cp, &buf); + _ = CaseFolding.caseFold(cp, &buf); return buf; } @@ -2628,8 +2627,6 @@ test "parse: sequences" { test "set: parseAndPut typical binding" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2653,8 +2650,6 @@ test "set: parseAndPut typical binding" { test "set: parseAndPut unconsumed binding" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2679,8 +2674,6 @@ test "set: parseAndPut unconsumed binding" { test "set: parseAndPut removed binding" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2699,8 +2692,6 @@ test "set: parseAndPut removed binding" { test "set: parseAndPut sequence" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2725,8 +2716,6 @@ test "set: parseAndPut sequence" { test "set: parseAndPut sequence with two actions" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2759,8 +2748,6 @@ test "set: parseAndPut sequence with two actions" { test "set: parseAndPut overwrite sequence" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2786,8 +2773,6 @@ test "set: parseAndPut overwrite sequence" { test "set: parseAndPut overwrite leader" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2813,8 +2798,6 @@ test "set: parseAndPut overwrite leader" { test "set: parseAndPut unbind sequence unbinds leader" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2831,8 +2814,6 @@ test "set: parseAndPut unbind sequence unbinds leader" { test "set: parseAndPut unbind sequence unbinds leader if not set" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2848,8 +2829,6 @@ test "set: parseAndPut unbind sequence unbinds leader if not set" { test "set: parseAndPut sequence preserves reverse mapping" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2867,8 +2846,6 @@ test "set: parseAndPut sequence preserves reverse mapping" { test "set: put overwrites sequence" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2889,8 +2866,6 @@ test "set: put overwrites sequence" { test "set: maintains reverse mapping" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2919,8 +2894,6 @@ test "set: maintains reverse mapping" { test "set: performable is not part of reverse mappings" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2954,8 +2927,6 @@ test "set: performable is not part of reverse mappings" { test "set: overriding a mapping updates reverse" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -2977,8 +2948,6 @@ test "set: overriding a mapping updates reverse" { test "set: consumed state" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -3004,8 +2973,6 @@ test "set: consumed state" { test "set: getEvent physical" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -3036,8 +3003,6 @@ test "set: getEvent physical" { test "set: getEvent codepoint" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -3078,8 +3043,6 @@ test "set: getEvent codepoint" { test "set: getEvent codepoint case folding" { const testing = std.testing; const alloc = testing.allocator; - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); var s: Set = .{}; defer s.deinit(alloc); @@ -3123,8 +3086,6 @@ test "Action: clone" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); const alloc = arena.allocator(); - _ = try @import("../global.zig").Zg.initForTesting(); - defer zg.deinitForTesting(); { var a: Action = .ignore; diff --git a/src/renderer/cell.zig b/src/renderer/cell.zig index 51abd7889..671890fbf 100644 --- a/src/renderer/cell.zig +++ b/src/renderer/cell.zig @@ -224,9 +224,9 @@ pub fn constraintWidth(cell_pin: terminal.Pin) u2 { const cell = cell_pin.rowAndCell().cell; const cp = cell.codepoint(); - if (!ziglyph.general_category.isPrivateUse(cp) and - !ziglyph.blocks.isDingbats(cp)) - { + // If it's not Private Use (Co) or Dingbats (0x2700-0x27bf), use grid + // width. + if (GeneralCategories.gc(cp) != .Co and !(cp >= 0x2700 and cp <= 0x27bf)) { return cell.gridWidth(); } diff --git a/src/unicode/main.zig b/src/unicode/main.zig index 03544e310..85c369168 100644 --- a/src/unicode/main.zig +++ b/src/unicode/main.zig @@ -8,10 +8,6 @@ pub const Properties = props.Properties; pub const graphemeBreak = grapheme.graphemeBreak; pub const GraphemeBreakState = grapheme.BreakState; -test { - @import("std").testing.refAllDecls(@This()); -} - /// Build Ghostty with `zig build -Doptimize=ReleaseFast -Demit-unicode-test`. /// /// Usage: ./zig-out/bin/unicode-test [width|class|break|all] [old|zg|ziglyph|all] @@ -41,9 +37,6 @@ pub fn main() !void { const args = try std.process.argsAlloc(alloc); defer std.process.argsFree(alloc, args); - var zg = try props.Context.init(alloc); - defer zg.deinit(alloc); - const ziglyph = @import("ziglyph"); const Graphemes = @import("Graphemes"); const DisplayWidth = @import("DisplayWidth"); @@ -74,14 +67,14 @@ pub fn main() !void { } if (compareZg) { - const zg_width = @min(2, @max(0, DisplayWidth.codePointWidth(zg.display_width, @intCast(cp)))); + const zg_width = @min(2, @max(0, DisplayWidth.codePointWidth(@intCast(cp)))); if (t.width != zg_width) { std.log.warn("[zg mismatch] cp={x} t={} zg={}", .{ cp, t.width, zg_width }); } } if (compareZiglyph) { - const ziglyph_width = @min(2, @max(0, DisplayWidth.codePointWidth(zg.display_width, @intCast(cp)))); + const ziglyph_width = @min(2, @max(0, DisplayWidth.codePointWidth(@intCast(cp)))); if (t.width != ziglyph_width) { std.log.warn("[ziglyph mismatch] cp={x} t={} zg={}", .{ cp, t.width, ziglyph_width }); } @@ -105,7 +98,7 @@ pub fn main() !void { } if (compareZg) { - const gbp = Graphemes.gbp(zg.graphemes, @intCast(cp)); + const gbp = Graphemes.gbp(@intCast(cp)); const matches = switch (t.grapheme_boundary_class) { .extended_pictographic_base => gbp == .Emoji_Modifier_Base, .emoji_modifier => gbp == .Emoji_Modifier, @@ -179,11 +172,11 @@ pub fn main() !void { if (cp1 % 0x100 == 0) std.log.info("progress: cp1={x}", .{cp1}); if (cp1 == '\r' or cp1 == '\n' or - Graphemes.gbp(zg.graphemes, @intCast(cp1)) == .Control) continue; + Graphemes.gbp(@intCast(cp1)) == .Control) continue; for (min..max) |cp2| { if (cp2 == '\r' or cp2 == '\n' or - Graphemes.gbp(zg.graphemes, @intCast(cp1)) == .Control) continue; + Graphemes.gbp(@intCast(cp1)) == .Control) continue; const gb = graphemeBreak(@intCast(cp1), @intCast(cp2), &state); @@ -202,7 +195,7 @@ pub fn main() !void { } if (compareZg) { - const zg_gb = Graphemes.graphemeBreak(@intCast(cp1), @intCast(cp2), &zg.graphemes, &zg_state); + const zg_gb = Graphemes.graphemeBreak(@intCast(cp1), @intCast(cp2), &zg_state); if (gb != zg_gb) { std.log.warn("[zg mismatch] cp1={x} cp2={x} gb={} zg_gb={} state={} zg_state={}", .{ cp1, diff --git a/src/unicode/props.zig b/src/unicode/props.zig index bfdcb3533..049f42ebd 100644 --- a/src/unicode/props.zig +++ b/src/unicode/props.zig @@ -1,33 +1,15 @@ const std = @import("std"); const assert = std.debug.assert; const lut = @import("lut.zig"); -const Graphemes = @import("Graphemes"); /// The context needed for lut generation. pub const Context = struct { - graphemes: Graphemes, - display_width: DisplayWidth, - // Whether to use the old implementation based on ziglyph. old: bool = false, + const Graphemes = @import("Graphemes"); const DisplayWidth = @import("DisplayWidth"); - // Public only for unicode-test - pub fn init(alloc: std.mem.Allocator) !Context { - const graphemes = try Graphemes.init(alloc); - return .{ - .graphemes = graphemes, - .display_width = try DisplayWidth.initWithGraphemes(alloc, graphemes), - }; - } - - // Public only for unicode-test - pub fn deinit(self: *Context, alloc: std.mem.Allocator) void { - self.graphemes.deinit(alloc); - self.display_width.deinit(alloc); - } - pub fn get(self: Context, cp: u21) !Properties { if (cp > 0x10FFFF) { return .{ @@ -35,12 +17,12 @@ pub const Context = struct { .grapheme_boundary_class = .invalid, }; } else { - const zg_width = DisplayWidth.codePointWidth(self.display_width, cp); + const zg_width = DisplayWidth.codePointWidth(cp); return .{ .width = @intCast(@min(2, @max(0, zg_width))), //.grapheme_boundary_class = .init(self, cp), - .grapheme_boundary_class = if (self.old) .initOld(cp) else .init(self, cp), + .grapheme_boundary_class = if (self.old) .initOld(cp) else .init(cp), }; } } @@ -137,10 +119,12 @@ pub const GraphemeBoundaryClass = enum(u4) { extended_pictographic_base, // \p{Extended_Pictographic} & \p{Emoji_Modifier_Base} emoji_modifier, // \p{Emoji_Modifier} + const Graphemes = @import("Graphemes"); + /// Gets the grapheme boundary class for a codepoint. This is VERY /// SLOW. The use case for this is only in generating lookup tables. - pub fn init(ctx: Context, cp: u21) GraphemeBoundaryClass { - return switch (Graphemes.gbp(ctx.graphemes, cp)) { + pub fn init(cp: u21) GraphemeBoundaryClass { + return switch (Graphemes.gbp(cp)) { .Emoji_Modifier_Base => .extended_pictographic_base, .Emoji_Modifier => .emoji_modifier, .Extended_Pictographic => .extended_pictographic, @@ -214,8 +198,7 @@ pub fn main() !void { const args = try std.process.argsAlloc(alloc); defer std.process.argsFree(alloc, args); - var ctx = try Context.init(alloc); - defer ctx.deinit(alloc); + var ctx = Context{}; if (args.len > 1 and std.mem.eql(u8, args[1], "old")) { ctx.old = true; From 265f2aa82ff9660a920197890832cfb930b2471e Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Mon, 7 Jul 2025 00:40:03 -0400 Subject: [PATCH 09/12] rest of changes, from stash --- src/unicode/lut.zig | 4 ++++ src/unicode/props.zig | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/unicode/lut.zig b/src/unicode/lut.zig index 84d51792d..9e6c2be16 100644 --- a/src/unicode/lut.zig +++ b/src/unicode/lut.zig @@ -177,3 +177,7 @@ pub fn Tables(comptime Elem: type) type { } }; } + +test { + std.testing.refAllDecls(@This()); +} diff --git a/src/unicode/props.zig b/src/unicode/props.zig index 049f42ebd..a14b0c448 100644 --- a/src/unicode/props.zig +++ b/src/unicode/props.zig @@ -222,3 +222,8 @@ pub fn main() !void { // t.stage3.len, // }); } + +test { + _ = table; + _ = Properties; +} From 5f80ba7b2cf833dae7f3996975894925c85bf515 Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Mon, 7 Jul 2025 00:53:05 -0400 Subject: [PATCH 10/12] fix bench and unicode-test builds --- src/build/GhosttyUnicodeTest.zig | 9 --------- src/build/SharedDeps.zig | 4 ++++ src/global.zig | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/build/GhosttyUnicodeTest.zig b/src/build/GhosttyUnicodeTest.zig index 4063c40ad..8421e9fae 100644 --- a/src/build/GhosttyUnicodeTest.zig +++ b/src/build/GhosttyUnicodeTest.zig @@ -36,15 +36,6 @@ pub fn init(b: *std.Build, cfg: *const Config, deps: *const SharedDeps) !Unicode exe.root_module.addImport("ziglyph", dep.module("ziglyph")); } - // Add zg dependencies used just for unicode-test - if (b.lazyDependency("zg", .{ - .target = cfg.target, - .optimize = cfg.optimize, - })) |dep| { - exe.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); - exe.root_module.addImport("Graphemes", dep.module("Graphemes")); - } - // Add the old version of the unicode tables const old_unicode_tables = try UnicodeTables.init(b); old_unicode_tables.run.addArg("old"); diff --git a/src/build/SharedDeps.zig b/src/build/SharedDeps.zig index 544be5c68..367baf5d2 100644 --- a/src/build/SharedDeps.zig +++ b/src/build/SharedDeps.zig @@ -415,6 +415,10 @@ pub fn add( .target = target, .optimize = optimize, })) |dep| { + if (self.config.emit_bench or self.config.emit_unicode_test) { + step.root_module.addImport("Graphemes", dep.module("Graphemes")); + step.root_module.addImport("DisplayWidth", dep.module("DisplayWidth")); + } step.root_module.addImport("CaseFolding", dep.module("CaseFolding")); step.root_module.addImport("Emoji", dep.module("Emoji")); step.root_module.addImport("GeneralCategories", dep.module("GeneralCategories")); diff --git a/src/global.zig b/src/global.zig index 668d2faec..ff84f3e9e 100644 --- a/src/global.zig +++ b/src/global.zig @@ -51,7 +51,7 @@ pub const GlobalState = struct { // defer { // const end = std.time.Instant.now() catch unreachable; // // "[updateFrame critical time] \t" - // std.log.err("[global init time] start={}us duration={}ns", .{ start_micro, end.since(start) / std.time.ns_per_us }); + // std.log.err("[global init time] start={}us duration={}us", .{ start_micro, end.since(start) / std.time.ns_per_us }); // } // Initialize ourself to nothing so we don't have any extra state. From 7c9b60dcafc70f70d34919cce466e2c33006eca9 Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Mon, 7 Jul 2025 00:57:01 -0400 Subject: [PATCH 11/12] update simd/codepoint_width.zig --- src/simd/codepoint_width.zig | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/simd/codepoint_width.zig b/src/simd/codepoint_width.zig index e94b9a300..bdb7c295c 100644 --- a/src/simd/codepoint_width.zig +++ b/src/simd/codepoint_width.zig @@ -4,9 +4,7 @@ const std = @import("std"); extern "c" fn ghostty_simd_codepoint_width(u32) i8; pub fn codepointWidth(cp: u32) i8 { - // const zg = try @import("../../global.zig").Zg.initForTesting(); - // defer zg.deinitForTesting(); - // try testing.expectEqual(@as(i8, 1), @import("DisplayWidth").codePointWidth(zg.display_width, @intCast(cp))); + // try testing.expectEqual(@as(i8, 1), @import("DisplayWidth").codePointWidth(@intCast(cp))); return ghostty_simd_codepoint_width(cp); } @@ -21,9 +19,7 @@ test "codepointWidth basic" { try testing.expectEqual(@as(i8, 2), codepointWidth(0xF900)); // 豈 try testing.expectEqual(@as(i8, 2), codepointWidth(0x20000)); // 𠀀 try testing.expectEqual(@as(i8, 2), codepointWidth(0x30000)); // 𠀀 - // const zg = try @import("../../global.zig").Zg.initForTesting(); - // defer zg.deinitForTesting(); - // try testing.expectEqual(@as(i8, 1), @import("DisplayWidth").codePointWidth(zg.display_width, 0x100)); + // try testing.expectEqual(@as(i8, 1), @import("DisplayWidth").codePointWidth(0x100)); } // This is not very fast in debug modes, so its commented by default. @@ -31,14 +27,12 @@ test "codepointWidth basic" { //test "codepointWidth matches zg" { // const testing = std.testing; // const DisplayWidth = @import("DisplayWidth"); -// const display_width = try DisplayWidth.init(std.testing.allocator); -// defer display_width.deinit(std.testing.allocator); // var success: bool = true; // // const min = 0xFF + 1; // start outside ascii // for (min..0x110000) |cp| { // const simd = codepointWidth(@intCast(cp)); -// const zg_width = DisplayWidth.codePointWidth(display_width, @intCast(cp)); +// const zg_width = DisplayWidth.codePointWidth(@intCast(cp)); // if (simd != zg_width) mismatch: { // if (cp == 0x2E3B) { // try testing.expectEqual(@as(i8, 2), simd); From d58dee96ee7e44b398077f53c200dc57c176f212 Mon Sep 17 00:00:00 2001 From: Jacob Sandlund Date: Mon, 7 Jul 2025 11:39:41 -0400 Subject: [PATCH 12/12] update zg to remove debug logging --- build.zig.zon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 7d9490ee4..7a2965d1b 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -42,8 +42,8 @@ .lazy = true, }, .zg = .{ - .url = "https://codeberg.org/jacobsandlund/zg/archive/bfb71988dbef3d11e54637fa7f99a84a2c7c9d1f.tar.gz", - .hash = "zg-0.14.0-oGqU3A-9sgKPomBsiqZsVKUayWJ2oIwolsb_6PwM7PnX", + .url = "https://codeberg.org/jacobsandlund/zg/archive/4d09cce1c40e0a704e5c1dfab1f4398f5c87b96b.tar.gz", + .hash = "zg-0.14.0-oGqU3Oi7sgLDn7I8RF43Fqg8hMbfVbCU5tTJvOZvcGV1", .lazy = true, }, .zig_wayland = .{