diff --git a/kitty/core_text.m b/kitty/core_text.m
index 0828de3be..4a1076adb 100644
--- a/kitty/core_text.m
+++ b/kitty/core_text.m
@@ -262,9 +262,9 @@ harfbuzz_font_for_face(PyObject *self) { return ((Face*)self)->harfbuzz_font; }
 
 bool
 set_size_for_face(PyObject *s, float pt_sz, float xdpi, float ydpi) {
-    return true;
     Face *self = (Face*)s;
     float dpi = (xdpi + ydpi) / 2.f;
+    if (self->dpi == dpi && self->point_sz == pt_sz) return true;
     CTFontRef f = CTFontCreateCopyWithAttributes(self->font, self->scaled_point_sz, NULL, NULL);
     if (f == NULL) { PyErr_SetString(PyExc_ValueError, "Failed to create font copy with different size"); return false; }
     CFRelease(self->font); self->font = NULL;
diff --git a/kitty/freetype.c b/kitty/freetype.c
index e73208a75..8edd96545 100644
--- a/kitty/freetype.c
+++ b/kitty/freetype.c
@@ -88,10 +88,12 @@ set_font_size(Face *self, FT_F26Dot6 char_width, FT_F26Dot6 char_height, FT_UInt
 }
 
 bool
-set_size_for_face(PyObject *self, float pt_sz, float xdpi, float ydpi) {
+set_size_for_face(PyObject *s, float pt_sz, float xdpi, float ydpi) {
+    Face *self = (Face*)s;
     FT_UInt w = (FT_UInt)(ceilf(pt_sz * 64));
+    if (self->char_width == w && self->char_height == w && self->xdpi == (FT_UInt)xdpi && self->ydpi == (FT_UInt)ydpi) return true;
     ((Face*)self)->size_in_pts = pt_sz;
-    return set_font_size((Face*)self, w, w, (FT_UInt)xdpi, (FT_UInt) ydpi);
+    return set_font_size(self, w, w, (FT_UInt)xdpi, (FT_UInt) ydpi);
 }
 
 static inline int