Reduce InstanceData footprint

The InstanceData type in the rendering subsystem was previously 17 f32s
plus one u8 which occupied a total of 72 bytes per instance. This meant
that for every character or background cell drawn, 72 bytes were sent to
the GPU. In the case of a 400x100 cell grid, a total of 2.9MB would be
sent.

This patch reduces InstanceData's size to 36 bytes, a 50% improvement!
Using the above example for comparison, a worst case of 1.44MB would be
transferred.

The motivation for this patch comes from macOS. Once the terminal grid
would reach a certain size, performance experienced a sharp and dramatic
drop (render times would go from ~3ms to ~16ms). I don't want to
speculate too much on the underlying issue, but suffice it to say that
this patch alleviates the problem in my testing.

While the performance impact was most significant on macOS, with
rendering times cut by more than 50% in some cases, this also results in
a measurable performance difference on other systems with high density
grids.

Co-authored-by: Christian Duerr <contact@christianduerr.com>

4 files changed, 99 insertions, 127 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4277034a..caeee1f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Fixed
 
 - Incorrect window location with negative `window.position` config options
+- Slow rendering performance with HiDPI displays, especially on macOS
 
 ## 0.5.0
 
diff --git a/alacritty/res/text.f.glsl b/alacritty/res/text.f.glsl
index cf477eb0..b051ee8d 100644
--- a/alacritty/res/text.f.glsl
+++ b/alacritty/res/text.f.glsl
@@ -13,9 +13,8 @@
 // limitations under the License.
 #version 330 core
 in vec2 TexCoords;
-flat in vec3 fg;
+flat in vec4 fg;
 flat in vec4 bg;
-flat in int colored;
 uniform int backgroundPass;
 
 layout(location = 0, index = 0) out vec4 color;
@@ -32,7 +31,7 @@ void main()
         alphaMask = vec4(1.0);
         color = vec4(bg.rgb, 1.0);
     } else {
-        if (colored != 0) {
+        if (fg.a != 0.0) {
             // Color glyphs, like emojis.
             vec4 glyphColor = texture(mask, TexCoords);
             alphaMask = vec4(glyphColor.a);
@@ -47,7 +46,7 @@ void main()
             // Regular text glyphs.
             vec3 textColor = texture(mask, TexCoords).rgb;
             alphaMask = vec4(textColor, textColor.r);
-            color = vec4(fg, 1.0);
+            color = vec4(fg.rgb, 1.0);
         }
     }
 }
diff --git a/alacritty/res/text.v.glsl b/alacritty/res/text.v.glsl
index 8978c111..d15728dc 100644
--- a/alacritty/res/text.v.glsl
+++ b/alacritty/res/text.v.glsl
@@ -21,19 +21,15 @@ layout (location = 1) in vec4 glyph;
 // uv mapping.
 layout (location = 2) in vec4 uv;
 
-// Text fg color.
-layout (location = 3) in vec3 textColor;
+// Text foreground rgb packed together with multicolor flag.
+layout (location = 3) in vec4 textColor;
 
 // Background color.
 layout (location = 4) in vec4 backgroundColor;
 
-// Set to 1 if the glyph colors should be kept.
-layout (location = 5) in int coloredGlyph;
-
 out vec2 TexCoords;
-flat out vec3 fg;
+flat out vec4 fg;
 flat out vec4 bg;
-flat out int colored;
 
 // Terminal properties
 uniform vec2 cellDim;
@@ -74,6 +70,5 @@ void main()
     }
 
     bg = vec4(backgroundColor.rgb / 255.0, backgroundColor.a);
-    fg = textColor / vec3(255.0, 255.0, 255.0);
-    colored = coloredGlyph;
+    fg = vec4(textColor.rgb / 255.0, textColor.a);
 }
diff --git a/alacritty/src/renderer/mod.rs b/alacritty/src/renderer/mod.rs
index 79df6b34..6b27a837 100644
--- a/alacritty/src/renderer/mod.rs
+++ b/alacritty/src/renderer/mod.rs
@@ -124,11 +124,11 @@ pub struct RectShaderProgram {
 #[derive(Copy, Debug, Clone)]
 pub struct Glyph {
     tex_id: GLuint,
-    colored: bool,
-    top: f32,
-    left: f32,
-    width: f32,
-    height: f32,
+    multicolor: u8,
+    top: i16,
+    left: i16,
+    width: i16,
+    height: i16,
     uv_bot: f32,
     uv_left: f32,
     uv_width: f32,
@@ -392,31 +392,31 @@ impl GlyphCache {
 #[repr(C)]
 struct InstanceData {
     // Coords.
-    col: f32,
-    row: f32,
+    col: u16,
+    row: u16,
     // Glyph offset.
-    left: f32,
-    top: f32,
-    // Glyph scale.
-    width: f32,
-    height: f32,
-    // uv offset.
+    left: i16,
+    top: i16,
+    // Glyph size.
+    width: i16,
+    height: i16,
+    // UV offset.
     uv_left: f32,
     uv_bot: f32,
     // uv scale.
     uv_width: f32,
     uv_height: f32,
     // Color.
-    r: f32,
-    g: f32,
-    b: f32,
-    // Background color.
-    bg_r: f32,
-    bg_g: f32,
-    bg_b: f32,
-    bg_a: f32,
-    // Flag indicating that glyph uses multiple colors, like an Emoji.
+    r: u8,
+    g: u8,
+    b: u8,
+    // Flag indicating that a glyph uses multiple colors; like an Emoji.
     multicolor: u8,
+    // Background color.
+    bg_r: u8,
+    bg_g: u8,
+    bg_b: u8,
+    bg_a: u8,
 }
 
 #[derive(Debug)]
@@ -471,8 +471,8 @@ impl Batch {
         }
 
         self.instances.push(InstanceData {
-            col: cell.column.0 as f32,
-            row: cell.line.0 as f32,
+            col: cell.column.0 as u16,
+            row: cell.line.0 as u16,
 
             top: glyph.top,
             left: glyph.left,
@@ -484,15 +484,15 @@ impl Batch {
             uv_width: glyph.uv_width,
             uv_height: glyph.uv_height,
 
-            r: f32::from(cell.fg.r),
-            g: f32::from(cell.fg.g),
-            b: f32::from(cell.fg.b),
+            r: cell.fg.r,
+            g: cell.fg.g,
+            b: cell.fg.b,
 
-            bg_r: f32::from(cell.bg.r),
-            bg_g: f32::from(cell.bg.g),
-            bg_b: f32::from(cell.bg.b),
-            bg_a: cell.bg_alpha,
-            multicolor: glyph.colored as u8,
+            bg_r: cell.bg.r,
+            bg_g: cell.bg.g,
+            bg_b: cell.bg.b,
+            bg_a: (cell.bg_alpha * 255.0) as u8,
+            multicolor: glyph.multicolor,
         });
     }
 
@@ -581,72 +581,49 @@ impl QuadRenderer {
                 ptr::null(),
                 gl::STREAM_DRAW,
             );
+
+            let mut index = 0;
+            let mut size = 0;
+
+            macro_rules! add_attr {
+                ($count:expr, $gl_type:expr, $type:ty) => {
+                    gl::VertexAttribPointer(
+                        index,
+                        $count,
+                        $gl_type,
+                        gl::FALSE,
+                        size_of::<InstanceData>() as i32,
+                        size as *const _,
+                    );
+                    gl::EnableVertexAttribArray(index);
+                    gl::VertexAttribDivisor(index, 1);
+
+                    #[allow(unused_assignments)]
+                    {
+                        size += $count * size_of::<$type>();
+                        index += 1;
+                    }
+                };
+            }
+
             // Coords.
-            gl::VertexAttribPointer(
-                0,
-                2,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                ptr::null(),
-            );
-            gl::EnableVertexAttribArray(0);
-            gl::VertexAttribDivisor(0, 1);
-            // Glyph offset.
-            gl::VertexAttribPointer(
-                1,
-                4,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (2 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(1);
-            gl::VertexAttribDivisor(1, 1);
-            // uv.
-            gl::VertexAttribPointer(
-                2,
-                4,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (6 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(2);
-            gl::VertexAttribDivisor(2, 1);
-            // Color.
-            gl::VertexAttribPointer(
-                3,
-                3,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (10 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(3);
-            gl::VertexAttribDivisor(3, 1);
+            add_attr!(2, gl::UNSIGNED_SHORT, u16);
+
+            // Glyph offset and size.
+            add_attr!(4, gl::SHORT, i16);
+
+            // UV offset.
+            add_attr!(4, gl::FLOAT, f32);
+
+            // Color and multicolor flag.
+            //
+            // These are packed together because of an OpenGL driver issue on macOS, which caused a
+            // `vec3(u8)` text color and a `u8` multicolor flag to increase the rendering time by a
+            // huge margin.
+            add_attr!(4, gl::UNSIGNED_BYTE, u8);
+
             // Background color.
-            gl::VertexAttribPointer(
-                4,
-                4,
-                gl::FLOAT,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (13 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(4);
-            gl::VertexAttribDivisor(4, 1);
-            // Multicolor flag.
-            gl::VertexAttribPointer(
-                5,
-                1,
-                gl::BYTE,
-                gl::FALSE,
-                size_of::<InstanceData>() as i32,
-                (17 * size_of::<f32>()) as *const _,
-            );
-            gl::EnableVertexAttribArray(5);
-            gl::VertexAttribDivisor(5, 1);
+            add_attr!(4, gl::UNSIGNED_BYTE, u8);
 
             // Rectangle setup.
             gl::GenVertexArrays(1, &mut rect_vao);
@@ -1089,7 +1066,7 @@ impl<'a> RenderApi<'a> {
             // right side of the preceding character. Since we render the
             // zero-width characters inside the preceding character, the
             // anchor has been moved to the right by one cell.
-            glyph.left += glyph_cache.metrics.average_advance as f32;
+            glyph.left += glyph_cache.metrics.average_advance as i16;
 
             self.add_render_item(cell, &glyph);
         }
@@ -1121,15 +1098,15 @@ fn load_glyph(
         },
         Err(AtlasInsertError::GlyphTooLarge) => Glyph {
             tex_id: atlas[*current_atlas].id,
-            colored: false,
-            top: 0.0,
-            left: 0.0,
-            width: 0.0,
-            height: 0.0,
-            uv_bot: 0.0,
-            uv_left: 0.0,
-            uv_width: 0.0,
-            uv_height: 0.0,
+            multicolor: 0,
+            top: 0,
+            left: 0,
+            width: 0,
+            height: 0,
+            uv_bot: 0.,
+            uv_left: 0.,
+            uv_width: 0.,
+            uv_height: 0.,
         },
     }
 }
@@ -1590,7 +1567,7 @@ impl Atlas {
         let offset_x = self.row_extent;
         let height = glyph.height as i32;
         let width = glyph.width as i32;
-        let colored;
+        let multicolor;
 
         unsafe {
             gl::BindTexture(gl::TEXTURE_2D, self.id);
@@ -1598,11 +1575,11 @@ impl Atlas {
             // Load data into OpenGL.
             let (format, buf) = match &glyph.buf {
                 BitmapBuffer::RGB(buf) => {
-                    colored = false;
+                    multicolor = false;
                     (gl::RGB, buf)
                 },
                 BitmapBuffer::RGBA(buf) => {
-                    colored = true;
+                    multicolor = true;
                     (gl::RGBA, buf)
                 },
             };
@@ -1637,11 +1614,11 @@ impl Atlas {
 
         Glyph {
             tex_id: self.id,
-            colored,
-            top: glyph.top as f32,
-            width: width as f32,
-            height: height as f32,
-            left: glyph.left as f32,
+            multicolor: multicolor as u8,
+            top: glyph.top as i16,
+            left: glyph.left as i16,
+            width: width as i16,
+            height: height as i16,
             uv_bot,
             uv_left,
             uv_width,