Merge pull request #2174 from ghostty-org/release-safe

Build ReleaseSafe builds for testing, make them more usable

.github/workflows/release-pr.yml

@@ -1,7 +1,4 @@
 on:
-  pull_request:
-    types: [opened, reopened, synchronize]
-
   workflow_dispatch: {}
 
 name: Release PR

.github/workflows/release-tip.yml

@@ -21,9 +21,9 @@ jobs:
           git tag -fa tip -m "Latest Continuous Release" ${GITHUB_SHA}
           git push --force origin tip
 
-  sentry-dsym-debug:
+  sentry-dsym-debug-slow:
     runs-on: namespace-profile-ghostty-sm
-    needs: [build-macos-debug]
+    needs: [build-macos-debug-slow]
     steps:
       - uses: actions/checkout@v4
 
@@ -34,7 +34,28 @@ jobs:
       - name: Download dSYM
         run: |
           GHOSTTY_COMMIT_LONG=$(git rev-parse HEAD)
-          curl -L https://tip.files.ghostty.dev/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug-dsym.zip > dsym.zip
+          curl -L https://tip.files.ghostty.dev/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug-slow-dsym.zip > dsym.zip
+
+      - name: Upload dSYM to Sentry
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+        run: |
+          sentry-cli dif upload --project ghostty --wait dsym.zip
+
+  sentry-dsym-debug-fast:
+    runs-on: namespace-profile-ghostty-sm
+    needs: [build-macos-debug-fast]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install sentry-cli
+        run: |
+          curl -sL https://sentry.io/get-cli/ | bash
+
+      - name: Download dSYM
+        run: |
+          GHOSTTY_COMMIT_LONG=$(git rev-parse HEAD)
+          curl -L https://tip.files.ghostty.dev/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug-fast-dsym.zip > dsym.zip
 
       - name: Upload dSYM to Sentry
         env:
@@ -244,7 +265,7 @@ jobs:
           source-dir: blob
           destination-dir: ./
 
-  build-macos-debug:
+  build-macos-debug-slow:
     if: |
       ${{
         github.event_name == 'workflow_dispatch' ||
@@ -382,8 +403,8 @@ jobs:
       - name: Zip App
         run: |
           cd macos/build/Release
-          zip -9 -r --symlinks ../../../ghostty-macos-universal-debug.zip Ghostty.app
-          zip -9 -r --symlinks ../../../ghostty-macos-universal-debug-dsym.zip Ghostty.app.dSYM/
+          zip -9 -r --symlinks ../../../ghostty-macos-universal-debug-slow.zip Ghostty.app
+          zip -9 -r --symlinks ../../../ghostty-macos-universal-debug-slow-dsym.zip Ghostty.app.dSYM/
 
       # Update Release
       - name: Release
@@ -393,7 +414,7 @@ jobs:
           prerelease: true
           tag_name: tip
           target_commitish: ${{ github.sha }}
-          files: ghostty-macos-universal-debug.zip
+          files: ghostty-macos-universal-debug-slow.zip
           token: ${{ secrets.GH_RELEASE_TOKEN }}
 
       # Update Blob Storage
@@ -401,8 +422,177 @@ jobs:
         run: |
           mkdir blob
           mkdir -p blob/${GHOSTTY_COMMIT_LONG}
-          cp ghostty-macos-universal-debug.zip blob/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug.zip
-          cp ghostty-macos-universal-debug-dsym.zip blob/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug-dsym.zip
+          cp ghostty-macos-universal-debug-slow.zip blob/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug-slow.zip
+          cp ghostty-macos-universal-debug-slow-dsym.zip blob/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug-slow-dsym.zip
+      - name: Upload to R2
+        uses: ryand56/r2-upload-action@latest
+        with:
+          r2-account-id: ${{ secrets.CF_R2_TIP_ACCOUNT_ID }}
+          r2-access-key-id: ${{ secrets.CF_R2_TIP_AWS_KEY }}
+          r2-secret-access-key: ${{ secrets.CF_R2_TIP_SECRET_KEY }}
+          r2-bucket: ghostty-tip
+          source-dir: blob
+          destination-dir: ./
+
+  build-macos-debug-fast:
+    if: |
+      ${{
+        github.event_name == 'workflow_dispatch' ||
+        (
+          github.event.workflow_run.conclusion == 'success' &&
+          github.repository_owner == 'ghostty-org' &&
+          github.ref_name == 'main'
+        )
+      }}
+
+    runs-on: namespace-profile-ghostty-macos
+    timeout-minutes: 90
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # Important so that build number generation works
+          fetch-depth: 0
+
+      # Install Nix and use that to run our tests so our environment matches exactly.
+      - uses: cachix/install-nix-action@V27
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - uses: cachix/cachix-action@v15
+        with:
+          name: ghostty
+          authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
+
+      # Setup Sparkle
+      - name: Setup Sparkle
+        env:
+          SPARKLE_VERSION: 2.5.1
+        run: |
+          mkdir -p .action/sparkle
+          cd .action/sparkle
+          curl -L https://github.com/sparkle-project/Sparkle/releases/download/${SPARKLE_VERSION}/Sparkle-for-Swift-Package-Manager.zip > sparkle.zip
+          unzip sparkle.zip
+          echo "$(pwd)/bin" >> $GITHUB_PATH
+
+      # Load Build Number
+      - name: Build Number
+        run: |
+          echo "GHOSTTY_BUILD=$(git rev-list --count head)" >> $GITHUB_ENV
+          echo "GHOSTTY_COMMIT=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          echo "GHOSTTY_COMMIT_LONG=$(git rev-parse HEAD)" >> $GITHUB_ENV
+
+      # GhosttyKit is the framework that is built from Zig for our native
+      # Mac app to access. Build this in release mode.
+      - name: Build GhosttyKit
+        run: nix develop -c zig build -Dstatic=true -Doptimize=ReleaseSafe
+
+      # The native app is built with native XCode tooling. This also does
+      # codesigning. IMPORTANT: this must NOT run in a Nix environment.
+      # Nix breaks xcodebuild so this has to be run outside.
+      - name: Build Ghostty.app
+        run: cd macos && xcodebuild -target Ghostty -configuration Release
+
+      # We inject the "build number" as simply the number of commits since HEAD.
+      # This will be a monotonically always increasing build number that we use.
+      - name: Update Info.plist
+        env:
+          SPARKLE_KEY_PUB: ${{ secrets.PROD_MACOS_SPARKLE_KEY_PUB }}
+        run: |
+          # Version Info
+          /usr/libexec/PlistBuddy -c "Set :GhosttyCommit $GHOSTTY_COMMIT" "macos/build/Release/Ghostty.app/Contents/Info.plist"
+          /usr/libexec/PlistBuddy -c "Set :CFBundleVersion $GHOSTTY_BUILD" "macos/build/Release/Ghostty.app/Contents/Info.plist"
+          /usr/libexec/PlistBuddy -c "Set :CFBundleShortVersionString $GHOSTTY_COMMIT" "macos/build/Release/Ghostty.app/Contents/Info.plist"
+
+          # Updater
+          /usr/libexec/PlistBuddy -c "Set :SUPublicEDKey $SPARKLE_KEY_PUB" "macos/build/Release/Ghostty.app/Contents/Info.plist"
+
+      - name: Codesign app bundle
+        env:
+          MACOS_CERTIFICATE: ${{ secrets.PROD_MACOS_CERTIFICATE }}
+          MACOS_CERTIFICATE_PWD: ${{ secrets.PROD_MACOS_CERTIFICATE_PWD }}
+          MACOS_CERTIFICATE_NAME: ${{ secrets.PROD_MACOS_CERTIFICATE_NAME }}
+          MACOS_CI_KEYCHAIN_PWD: ${{ secrets.PROD_MACOS_CI_KEYCHAIN_PWD }}
+        run: |
+          # Turn our base64-encoded certificate back to a regular .p12 file
+          echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
+
+          # We need to create a new keychain, otherwise using the certificate will prompt
+          # with a UI dialog asking for the certificate password, which we can't
+          # use in a headless CI environment
+          security create-keychain -p "$MACOS_CI_KEYCHAIN_PWD" build.keychain
+          security default-keychain -s build.keychain
+          security unlock-keychain -p "$MACOS_CI_KEYCHAIN_PWD" build.keychain
+          security import certificate.p12 -k build.keychain -P "$MACOS_CERTIFICATE_PWD" -T /usr/bin/codesign
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$MACOS_CI_KEYCHAIN_PWD" build.keychain
+
+          # Codesign Sparkle. Some notes here:
+          #   - The XPC services aren't used since we don't sandbox Ghostty,
+          #     but since they're part of the build, they still need to be
+          #     codesigned.
+          #   - The binaries in the "Versions" folders need to NOT be symlinks.
+          /usr/bin/codesign --verbose -f -s "$MACOS_CERTIFICATE_NAME" -o runtime "macos/build/Release/Ghostty.app/Contents/Frameworks/Sparkle.framework/Versions/B/XPCServices/Downloader.xpc"
+          /usr/bin/codesign --verbose -f -s "$MACOS_CERTIFICATE_NAME" -o runtime "macos/build/Release/Ghostty.app/Contents/Frameworks/Sparkle.framework/Versions/B/XPCServices/Installer.xpc"
+          /usr/bin/codesign --verbose -f -s "$MACOS_CERTIFICATE_NAME" -o runtime "macos/build/Release/Ghostty.app/Contents/Frameworks/Sparkle.framework/Versions/B/Autoupdate"
+          /usr/bin/codesign --verbose -f -s "$MACOS_CERTIFICATE_NAME" -o runtime "macos/build/Release/Ghostty.app/Contents/Frameworks/Sparkle.framework/Versions/B/Updater.app"
+          /usr/bin/codesign --verbose -f -s "$MACOS_CERTIFICATE_NAME" -o runtime "macos/build/Release/Ghostty.app/Contents/Frameworks/Sparkle.framework"
+
+          # Codesign the app bundle
+          /usr/bin/codesign --verbose -f -s "$MACOS_CERTIFICATE_NAME" -o runtime --entitlements "macos/Ghostty.entitlements" macos/build/Release/Ghostty.app
+
+      - name: "Notarize app bundle"
+        env:
+          PROD_MACOS_NOTARIZATION_APPLE_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_APPLE_ID }}
+          PROD_MACOS_NOTARIZATION_TEAM_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_TEAM_ID }}
+          PROD_MACOS_NOTARIZATION_PWD: ${{ secrets.PROD_MACOS_NOTARIZATION_PWD }}
+        run: |
+          # Store the notarization credentials so that we can prevent a UI password dialog
+          # from blocking the CI
+          echo "Create keychain profile"
+          xcrun notarytool store-credentials "notarytool-profile" --apple-id "$PROD_MACOS_NOTARIZATION_APPLE_ID" --team-id "$PROD_MACOS_NOTARIZATION_TEAM_ID" --password "$PROD_MACOS_NOTARIZATION_PWD"
+
+          # We can't notarize an app bundle directly, but we need to compress it as an archive.
+          # Therefore, we create a zip file containing our app bundle, so that we can send it to the
+          # notarization service
+          echo "Creating temp notarization archive"
+          ditto -c -k --keepParent "macos/build/Release/Ghostty.app" "notarization.zip"
+
+          # Here we send the notarization request to the Apple's Notarization service, waiting for the result.
+          # This typically takes a few seconds inside a CI environment, but it might take more depending on the App
+          # characteristics. Visit the Notarization docs for more information and strategies on how to optimize it if
+          # you're curious
+          echo "Notarize app"
+          xcrun notarytool submit "notarization.zip" --keychain-profile "notarytool-profile" --wait
+
+          # Finally, we need to "attach the staple" to our executable, which will allow our app to be
+          # validated by macOS even when an internet connection is not available.
+          echo "Attach staple"
+          xcrun stapler staple "macos/build/Release/Ghostty.app"
+
+      # Zip up the app
+      - name: Zip App
+        run: |
+          cd macos/build/Release
+          zip -9 -r --symlinks ../../../ghostty-macos-universal-debug-fast.zip Ghostty.app
+          zip -9 -r --symlinks ../../../ghostty-macos-universal-debug-fast-dsym.zip Ghostty.app.dSYM/
+
+      # Update Release
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: 'Ghostty Tip ("Nightly")'
+          prerelease: true
+          tag_name: tip
+          target_commitish: ${{ github.sha }}
+          files: ghostty-macos-universal-debug-fast.zip
+          token: ${{ secrets.GH_RELEASE_TOKEN }}
+
+      # Update Blob Storage
+      - name: Prep R2 Storage
+        run: |
+          mkdir blob
+          mkdir -p blob/${GHOSTTY_COMMIT_LONG}
+          cp ghostty-macos-universal-debug-fast.zip blob/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug-fast.zip
+          cp ghostty-macos-universal-debug-fast-dsym.zip blob/${GHOSTTY_COMMIT_LONG}/ghostty-macos-universal-debug-fast-dsym.zip
       - name: Upload to R2
         uses: ryand56/r2-upload-action@latest
         with:

macos/Sources/Features/Terminal/TerminalView.swift

@@ -80,7 +80,7 @@ struct TerminalView<ViewModel: TerminalViewModel>: View {
             VStack(spacing: 0) {
                 // If we're running in debug mode we show a warning so that users
                 // know that performance will be degraded.
-                if (Ghostty.info.mode == GHOSTTY_BUILD_MODE_DEBUG) {
+                if (Ghostty.info.mode == GHOSTTY_BUILD_MODE_DEBUG || Ghostty.info.mode == GHOSTTY_BUILD_MODE_RELEASE_SAFE) {
                     DebugBuildWarningView()
                 }
 

src/apprt/gtk/Window.zig

@@ -135,7 +135,7 @@ pub fn init(self: *Window, app: *App) !void {
 
     // In debug we show a warning. This is a really common issue where
     // people build from source in debug and performance is really bad.
-    if (builtin.mode == .Debug) {
+    if (comptime std.debug.runtime_safety) {
         const warning = c.gtk_label_new("⚠️ You're running a debug build of Ghostty! Performance will be degraded.");
         c.gtk_widget_set_margin_top(warning, 10);
         c.gtk_widget_set_margin_bottom(warning, 10);

src/build_config.zig

@@ -104,6 +104,19 @@ pub const app_runtime: apprt.Runtime = config.app_runtime;
 pub const font_backend: font.Backend = config.font_backend;
 pub const renderer: rendererpkg.Impl = config.renderer;
 
+/// True if we should have "slow" runtime safety checks. The initial motivation
+/// for this was terminal page/pagelist integrity checks. These were VERY
+/// slow but very thorough. But they made it so slow that the terminal couldn't
+/// be used for real work. We'd love to have an option to run a build with
+/// safety checks that could be used for real work. This lets us do that.
+pub const slow_runtime_safety = std.debug.runtime_safety and switch (builtin.mode) {
+    .Debug => true,
+    .ReleaseSafe,
+    .ReleaseSmall,
+    .ReleaseFast,
+    => false,
+};
+
 pub const Artifact = enum {
     /// Standalone executable
     exe,

src/terminal/PageList.zig

@@ -4,6 +4,7 @@
 const PageList = @This();
 
 const std = @import("std");
+const build_config = @import("../build_config.zig");
 const Allocator = std.mem.Allocator;
 const assert = std.debug.assert;
 const color = @import("color.zig");
@@ -1293,7 +1294,7 @@ fn resizeWithoutReflow(self: *PageList, opts: Resize) !void {
             },
         }
 
-        if (comptime std.debug.runtime_safety) {
+        if (build_config.slow_runtime_safety) {
             assert(self.totalRows() >= self.rows);
         }
     }
@@ -2301,7 +2302,7 @@ pub fn pin(self: *const PageList, pt: point.Point) ?Pin {
 /// pin points to is removed completely, the tracked pin will be updated
 /// to the top-left of the screen.
 pub fn trackPin(self: *PageList, p: Pin) Allocator.Error!*Pin {
-    if (comptime std.debug.runtime_safety) assert(self.pinIsValid(p));
+    if (build_config.slow_runtime_safety) assert(self.pinIsValid(p));
 
     // Create our tracked pin
     const tracked = try self.pool.pins.create();
@@ -2963,7 +2964,7 @@ pub fn pageIterator(
     else
         self.getBottomRight(tl_pt) orelse return .{ .row = null };
 
-    if (comptime std.debug.runtime_safety) {
+    if (build_config.slow_runtime_safety) {
         assert(tl_pin.eql(bl_pin) or tl_pin.before(bl_pin));
     }
 
@@ -3279,7 +3280,7 @@ pub const Pin = struct {
     // Note: this is primarily unit tested as part of the Kitty
     // graphics deletion code.
     pub fn isBetween(self: Pin, top: Pin, bottom: Pin) bool {
-        if (comptime std.debug.runtime_safety) {
+        if (build_config.slow_runtime_safety) {
             if (top.page == bottom.page) {
                 // If top is bottom, must be ordered.
                 assert(top.y <= bottom.y);

src/terminal/Screen.zig

@@ -1,6 +1,7 @@
 const Screen = @This();
 
 const std = @import("std");
+const build_config = @import("../build_config.zig");
 const Allocator = std.mem.Allocator;
 const assert = std.debug.assert;
 const ansi = @import("ansi.zig");
@@ -253,7 +254,7 @@ pub fn deinit(self: *Screen) void {
 /// tests. This only asserts the screen specific data so callers should
 /// ensure they're also calling page integrity checks if necessary.
 pub fn assertIntegrity(self: *const Screen) void {
-    if (comptime std.debug.runtime_safety) {
+    if (build_config.slow_runtime_safety) {
         assert(self.cursor.x < self.pages.cols);
         assert(self.cursor.y < self.pages.rows);
 
@@ -714,7 +715,7 @@ pub fn cursorDownScroll(self: *Screen) !void {
 
         // These assertions help catch some pagelist math errors. Our
         // x/y should be unchanged after the grow.
-        if (comptime std.debug.runtime_safety) {
+        if (build_config.slow_runtime_safety) {
             const active = self.pages.pointFromPin(
                 .active,
                 page_pin,

src/terminal/Terminal.zig

@@ -1149,10 +1149,8 @@ pub fn index(self: *Terminal) !void {
         // up by 1, so we need to move it back down. A `cursorReload`
         // would be better option but this is more efficient and this is
         // a super hot path so we do this instead.
-        if (comptime std.debug.runtime_safety) {
-            assert(self.screen.cursor.x == old_cursor.x);
-            assert(self.screen.cursor.y == old_cursor.y);
-        }
+        assert(self.screen.cursor.x == old_cursor.x);
+        assert(self.screen.cursor.y == old_cursor.y);
         self.screen.cursor.y -= 1;
         self.screen.cursorDown(1);
 

src/terminal/page.zig

@@ -1,5 +1,6 @@
 const std = @import("std");
 const builtin = @import("builtin");
+const build_config = @import("../build_config.zig");
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
 const assert = std.debug.assert;
@@ -181,8 +182,8 @@ pub const Page = struct {
 
     /// If this is true then verifyIntegrity will do nothing. This is
     /// only present with runtime safety enabled.
-    pause_integrity_checks: if (std.debug.runtime_safety) usize else void =
-        if (std.debug.runtime_safety) 0 else {},
+    pause_integrity_checks: if (build_config.slow_runtime_safety) usize else void =
+        if (build_config.slow_runtime_safety) 0 else {},
 
     /// Initialize a new page, allocating the required backing memory.
     /// The size of the initialized page defaults to the full capacity.
@@ -305,7 +306,7 @@ pub const Page = struct {
     /// doing a lot of operations that would trigger integrity check
     /// violations but you know the page will end up in a consistent state.
     pub fn pauseIntegrityChecks(self: *Page, v: bool) void {
-        if (comptime std.debug.runtime_safety) {
+        if (build_config.slow_runtime_safety) {
             if (v) {
                 self.pause_integrity_checks += 1;
             } else {
@@ -318,9 +319,10 @@ pub const Page = struct {
     /// when runtime safety is enabled. This is a no-op when runtime
     /// safety is disabled. This uses the libc allocator.
     pub fn assertIntegrity(self: *const Page) void {
-        if (comptime std.debug.runtime_safety) {
-            self.verifyIntegrity(std.heap.c_allocator) catch unreachable;
-        }
+        self.verifyIntegrity(std.heap.c_allocator) catch |err| {
+            log.err("page integrity violation, crashing. err={}", .{err});
+            @panic("page integrity violation");
+        };
     }
 
     /// Verifies the integrity of the page data. This is not fast,
@@ -341,7 +343,7 @@ pub const Page = struct {
         //   used for the same reason as styles above.
         //
 
-        if (comptime std.debug.runtime_safety) {
+        if (build_config.slow_runtime_safety) {
             if (self.pause_integrity_checks > 0) return;
         }
 
@@ -737,7 +739,7 @@ pub const Page = struct {
             // This is an integrity check: if the row claims it doesn't
             // have managed memory then all cells must also not have
             // managed memory.
-            if (comptime std.debug.runtime_safety) {
+            if (build_config.slow_runtime_safety) {
                 for (other_cells) |cell| {
                     assert(!cell.hasGrapheme());
                     assert(!cell.hyperlink);
@@ -764,7 +766,7 @@ pub const Page = struct {
                 if (src_cell.hasGrapheme()) {
                     // To prevent integrity checks flipping. This will
                     // get fixed up when we check the style id below.
-                    if (comptime std.debug.runtime_safety) {
+                    if (build_config.slow_runtime_safety) {
                         dst_cell.style_id = style.default_id;
                     }
 
@@ -895,7 +897,7 @@ pub const Page = struct {
 
     /// Get the cells for a row.
     pub fn getCells(self: *const Page, row: *Row) []Cell {
-        if (comptime std.debug.runtime_safety) {
+        if (build_config.slow_runtime_safety) {
             const rows = self.rows.ptr(self.memory);
             const cells = self.cells.ptr(self.memory);
             assert(@intFromPtr(row) >= @intFromPtr(rows));
@@ -1244,7 +1246,7 @@ pub const Page = struct {
     pub fn appendGrapheme(self: *Page, row: *Row, cell: *Cell, cp: u21) Allocator.Error!void {
         defer self.assertIntegrity();
 
-        if (comptime std.debug.runtime_safety) assert(cell.codepoint() != 0);
+        if (build_config.slow_runtime_safety) assert(cell.codepoint() != 0);
 
         const cell_offset = getOffset(Cell, self.memory, cell);
         var map = self.grapheme_map.map(self.memory);
@@ -1317,7 +1319,7 @@ pub const Page = struct {
     /// there are scenarios where we want to move graphemes without changing
     /// the content tag. Callers beware but assertIntegrity should catch this.
     fn moveGrapheme(self: *Page, src: *Cell, dst: *Cell) void {
-        if (comptime std.debug.runtime_safety) {
+        if (build_config.slow_runtime_safety) {
             assert(src.hasGrapheme());
             assert(!dst.hasGrapheme());
         }
@@ -1334,7 +1336,7 @@ pub const Page = struct {
     /// Clear the graphemes for a given cell.
     pub fn clearGrapheme(self: *Page, row: *Row, cell: *Cell) void {
         defer self.assertIntegrity();
-        if (comptime std.debug.runtime_safety) assert(cell.hasGrapheme());
+        if (build_config.slow_runtime_safety) assert(cell.hasGrapheme());
 
         // Get our entry in the map, which must exist
         const cell_offset = getOffset(Cell, self.memory, cell);