macOS: Fix finding fallback font for private use unicode symbols not working reliably

Fixes #1650

docs/changelog.rst

@@ -32,6 +32,9 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
 - Linux: Fix incorrect scaling for fallback fonts when the font has an
   underscore that renders out of bounds (:iss:`1713`)
 
+- macOS: Fix finding fallback font for private use unicode symbols not working
+  reliably (:iss:`1650`)
+
 
 0.14.2 [2019-06-09]
 ---------------------

kitty/core_text.m

@@ -7,6 +7,7 @@
 
 #include "state.h"
 #include "fonts.h"
+#include "unicode-data.h"
 #include <structmember.h>
 #include <stdint.h>
 #include <math.h>
@@ -144,23 +145,79 @@ font_descriptor_from_python(PyObject *src) {
     return CTFontDescriptorCreateWithAttributes((CFDictionaryRef) attrs);
 }
 
+static CTFontCollectionRef all_fonts_collection_data = NULL;
+
+static CTFontCollectionRef
+all_fonts_collection() {
+    if (all_fonts_collection_data == NULL) all_fonts_collection_data = CTFontCollectionCreateFromAvailableFonts(NULL);
+    return all_fonts_collection_data;
+}
+
 static PyObject*
 coretext_all_fonts(PyObject UNUSED *_self) {
-    static CTFontCollectionRef collection = NULL;
-    if (collection == NULL) collection = CTFontCollectionCreateFromAvailableFonts(NULL);
-    NSArray *matches = (NSArray *) CTFontCollectionCreateMatchingFontDescriptors(collection);
-    PyObject *ans = PyTuple_New([matches count]), *temp;
+    CFArrayRef matches = CTFontCollectionCreateMatchingFontDescriptors(all_fonts_collection());
+    const CFIndex count = CFArrayGetCount(matches);
+    PyObject *ans = PyTuple_New(count), *temp;
     if (ans == NULL) return PyErr_NoMemory();
-    for (unsigned int i = 0; i < [matches count]; i++) {
-        temp = font_descriptor_to_python((CTFontDescriptorRef) matches[i]);
+    for (CFIndex i = 0; i < count; i++) {
+        temp = font_descriptor_to_python((CTFontDescriptorRef) CFArrayGetValueAtIndex(matches, i));
         if (temp == NULL) { Py_DECREF(ans); return NULL; }
         PyTuple_SET_ITEM(ans, i, temp); temp = NULL;
     }
     return ans;
 }
 
+static inline unsigned int
+glyph_id_for_codepoint_ctfont(CTFontRef ct_font, char_type ch) {
+    unichar chars[2] = {0};
+    CGGlyph glyphs[2] = {0};
+    int count = CFStringGetSurrogatePairForLongCharacter(ch, chars) ? 2 : 1;
+    CTFontGetGlyphsForCharacters(ct_font, chars, glyphs, count);
+    return glyphs[0];
+}
+
+static inline bool
+is_last_resort_font(CTFontRef new_font) {
+    CFStringRef name = CTFontCopyPostScriptName(new_font);
+    CFComparisonResult cr = CFStringCompare(name, CFSTR("LastResort"), 0);
+    CFRelease(name);
+    return cr == kCFCompareEqualTo;
+}
+
 static inline CTFontRef
-find_substitute_face(CFStringRef str, CTFontRef old_font) {
+manually_search_fallback_fonts(CTFontRef current_font, CPUCell *cell) {
+    CFArrayRef fonts = CTFontCollectionCreateMatchingFontDescriptors(all_fonts_collection());
+    CTFontRef ans = NULL;
+    const CFIndex count = CFArrayGetCount(fonts);
+    for (CFIndex i = 0; i < count; i++) {
+        CTFontDescriptorRef descriptor = (CTFontDescriptorRef)CFArrayGetValueAtIndex(fonts, i);
+        CTFontRef new_font = CTFontCreateWithFontDescriptor(descriptor, CTFontGetSize(current_font), NULL);
+        if (new_font) {
+            if (!is_last_resort_font(new_font)) {
+                char_type ch = cell->ch ? cell->ch : ' ';
+                bool found = true;
+                if (!glyph_id_for_codepoint_ctfont(new_font, ch)) found = false;
+                if (found && cell->cc_idx[0]) {
+                    ch = codepoint_for_mark(cell->cc_idx[0]);
+                    if (!glyph_id_for_codepoint_ctfont(new_font, ch)) found = false;
+                }
+                if (found && cell->cc_idx[1]) {
+                    ch = codepoint_for_mark(cell->cc_idx[1]);
+                    if (!glyph_id_for_codepoint_ctfont(new_font, ch)) found = false;
+                }
+                if (found) {
+                    ans = new_font;
+                    break;
+                }
+            }
+            CFRelease(new_font);
+        }
+    }
+    return ans;
+}
+
+static inline CTFontRef
+find_substitute_face(CFStringRef str, CTFontRef old_font, CPUCell *cpu_cell) {
     // CTFontCreateForString returns the original font when there are combining
     // diacritics in the font and the base character is in the original font,
     // so we have to check each character individually
@@ -171,11 +228,13 @@ find_substitute_face(CFStringRef str, CTFontRef old_font) {
         else start++;
         if (new_font == NULL) { PyErr_SetString(PyExc_ValueError, "Failed to find fallback CTFont"); return NULL; }
         if (new_font == old_font) { CFRelease(new_font); continue; }
-        CFStringRef name = CTFontCopyPostScriptName(new_font);
-        CFComparisonResult cr = CFStringCompare(name, CFSTR("LastResort"), 0);
-        CFRelease(name);
-        if (cr == kCFCompareEqualTo) {
+        if (is_last_resort_font(new_font)) {
             CFRelease(new_font);
+            if (is_private_use(cpu_cell->ch)) {
+                // CoreTexts fallback font mechanism does not work for private use characters
+                new_font = manually_search_fallback_fonts(old_font, cpu_cell);
+                if (new_font) return new_font;
+            }
             PyErr_SetString(PyExc_ValueError, "Failed to find fallback CTFont other than the LastResort font");
             return NULL;
         }
@@ -191,11 +250,11 @@ create_fallback_face(PyObject *base_face, CPUCell* cell, bool UNUSED bold, bool
     CTFontRef new_font;
     if (emoji_presentation) new_font = CTFontCreateWithName((CFStringRef)@"AppleColorEmoji", self->scaled_point_sz, NULL);
     else {
-        char text[256] = {0};
+        char text[64] = {0};
         cell_as_utf8_for_fallback(cell, text);
         CFStringRef str = CFStringCreateWithCString(NULL, text, kCFStringEncodingUTF8);
         if (str == NULL) return PyErr_NoMemory();
-        new_font = find_substitute_face(str, self->ct_font);
+        new_font = find_substitute_face(str, self->ct_font, cell);
         CFRelease(str);
     }
     if (new_font == NULL) return NULL;
@@ -205,11 +264,7 @@ create_fallback_face(PyObject *base_face, CPUCell* cell, bool UNUSED bold, bool
 unsigned int
 glyph_id_for_codepoint(PyObject *s, char_type ch) {
     CTFace *self = (CTFace*)s;
-    unichar chars[2] = {0};
-    CGGlyph glyphs[2] = {0};
-    int count = CFStringGetSurrogatePairForLongCharacter(ch, chars) ? 2 : 1;
-    CTFontGetGlyphsForCharacters(self->ct_font, chars, glyphs, count);
-    return glyphs[0];
+    return glyph_id_for_codepoint_ctfont(self->ct_font, ch);
 }
 
 bool
@@ -336,6 +391,7 @@ static CGPoint positions[128];
 static void
 finalize(void) {
     free(render_buf);
+    if (all_fonts_collection_data) CFRelease(all_fonts_collection_data);
 }
 
 

kitty/fonts.c

@@ -650,11 +650,6 @@ extract_cell_from_canvas(FontGroup *fg, unsigned int i, unsigned int num_cells)
     return ans;
 }
 
-static inline bool
-is_private_use(char_type ch) {
-    return (0xe000 <= ch && ch <= 0xf8ff) || (0xF0000 <= ch && ch <= 0xFFFFF) || (0x100000 <= ch && ch <= 0x10FFFF);
-}
-
 static inline void
 render_group(FontGroup *fg, unsigned int num_cells, unsigned int num_glyphs, CPUCell *cpu_cells, GPUCell *gpu_cells, hb_glyph_info_t *info, hb_glyph_position_t *positions, Font *font, glyph_index glyph, ExtraGlyphs *extra_glyphs, bool center_glyph) {
     static SpritePosition* sprite_position[16];

kitty/unicode-data.h

@@ -24,3 +24,8 @@ can_strip_from_end_of_url(uint32_t ch) {
         ch == '>'
     ) ? true : false;
 }
+
+static inline bool
+is_private_use(char_type ch) {
+    return (0xe000 <= ch && ch <= 0xf8ff) || (0xF0000 <= ch && ch <= 0xFFFFF) || (0x100000 <= ch && ch <= 0x10FFFF);
+}