diff --git a/kitty/boss.py b/kitty/boss.py
index de5246e4b..2a2cca55a 100644
--- a/kitty/boss.py
+++ b/kitty/boss.py
@@ -19,13 +19,12 @@ from .constants import (
     appname, config_dir, editor, set_boss, supports_primary_selection
 )
 from .fast_data_types import (
-    ChildMonitor, background_opacity_of, cell_size_for_window,
-    change_background_opacity, create_os_window, current_os_window,
-    destroy_global_data, get_clipboard_string, glfw_post_empty_event,
-    mark_os_window_for_close, set_clipboard_string,
+    ChildMonitor, background_opacity_of, change_background_opacity,
+    create_os_window, current_os_window, destroy_global_data,
+    get_clipboard_string, glfw_post_empty_event, global_font_size,
+    mark_os_window_for_close, os_window_font_size, set_clipboard_string,
     set_in_sequence_mode, show_window, toggle_fullscreen
 )
-from .fonts.render import resize_fonts
 from .keys import get_shortcut, shortcut_matches
 from .remote_control import handle_cmd
 from .rgb import Color, color_from_int
@@ -96,7 +95,6 @@ class Boss:
             talk_fd, listen_fd
         )
         set_boss(self)
-        self.current_font_size = opts.font_size
         self.opts, self.args = opts, args
         startup_session = create_session(opts, args)
         self.add_os_window(startup_session, os_window_id=os_window_id)
@@ -311,41 +309,67 @@ class Boss:
         if tm is not None:
             tm.resize()
 
-    def increase_font_size(self):
-        self.set_font_size(
-            min(
-                self.opts.font_size * 5, self.current_font_size +
-                self.opts.font_size_delta))
+    def increase_font_size(self):  # legacy
+        self.set_font_size(min(self.opts.font_size * 5, self.current_font_size + 2.0))
 
-    def decrease_font_size(self):
+    def decrease_font_size(self):  # legacy
         self.set_font_size(self.current_font_size - self.opts.font_size_delta)
 
-    def restore_font_size(self):
+    def restore_font_size(self):  # legacy
         self.set_font_size(self.opts.font_size)
 
-    def _change_font_size(self, new_size=None, on_dpi_change=False):
-        if new_size is not None:
-            self.current_font_size = new_size
-        windows = tuple(filter(None, self.window_id_map.values()))
-        old_sz_map = {w.id: cell_size_for_window(w.os_window_id) for w in windows}
-        resize_fonts(self.current_font_size, on_dpi_change=on_dpi_change)
-        for window in windows:
-            old_cell_width, old_cell_height = old_sz_map[window.id]
-            window.screen.rescale_images(old_cell_width, old_cell_height)
-            window.screen.refresh_sprite_positions()
-        for tm in self.os_window_map.values():
-            tm.resize()
-            tm.refresh_sprite_positions()
-        glfw_post_empty_event()
+    def set_font_size(self, new_size):  # legacy
+        self.change_font_size(True, None, new_size)
 
-    def set_font_size(self, new_size):
-        new_size = max(MINIMUM_FONT_SIZE, new_size)
-        if new_size == self.current_font_size:
-            return
-        self._change_font_size(new_size)
+    def change_font_size(self, all_windows, increment_operation, amt):
+        def calc_new_size(old_size):
+            new_size = old_size
+            if amt == 0:
+                new_size = self.opts.font_size
+            else:
+                if increment_operation:
+                    new_size += (1 if increment_operation == '+' else -1) * amt
+                else:
+                    new_size = amt
+                new_size = max(MINIMUM_FONT_SIZE, min(new_size, self.opts.font_size * 5))
+            return new_size
+
+        if all_windows:
+            current_global_size = global_font_size()
+            new_size = calc_new_size(current_global_size)
+            if new_size != current_global_size:
+                global_font_size(new_size)
+            os_windows = tuple(self.os_window_map.keys())
+        else:
+            os_windows = []
+            w = self.active_window
+            if w is not None:
+                os_windows.append(w.os_window_id)
+        if os_windows:
+            final_windows = {}
+            for wid in os_windows:
+                current_size = os_window_font_size(wid)
+                if current_size:
+                    new_size = calc_new_size(current_size)
+                    if new_size != current_size:
+                        final_windows[wid] = new_size
+            if final_windows:
+                self._change_font_size(final_windows)
+
+    def _change_font_size(self, sz_map):
+        for os_window_id, sz in sz_map.items():
+            tm = self.os_window_map.get(os_window_id)
+            if tm is not None:
+                os_window_font_size(os_window_id, sz)
+                tm.resize()
 
     def on_dpi_change(self, os_window_id):
-        self._change_font_size()
+        tm = self.os_window_map.get(os_window_id)
+        if tm is not None:
+            sz = os_window_font_size(os_window_id)
+            if sz:
+                os_window_font_size(os_window_id, sz, True)
+                tm.resize()
 
     def _set_os_window_background_opacity(self, os_window_id, opacity):
         change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0)))
diff --git a/kitty/config.py b/kitty/config.py
index 2ce059c5d..882c429bf 100644
--- a/kitty/config.py
+++ b/kitty/config.py
@@ -149,6 +149,22 @@ def float_parse(func, rest):
     return func, (float(rest),)
 
 
+@func_with_args('change_font_size')
+def parse_change_font_size(func, rest):
+    vals = rest.split(' ', 1)
+    if len(vals) != 2:
+        log_error('Invalid change_font_size specification: {}, treating it as default'.format(rest))
+        args = [True, None, 0]
+    else:
+        args = [vals[0].lower() == 'all', None, 0]
+        amt = vals[1]
+        if amt[0] in '+-':
+            args[1] = amt[0]
+            amt = amt[1:]
+        args[2] = float(amt)
+    return func, args
+
+
 def parse_key_action(action):
     parts = action.split(' ', 1)
     func = parts[0]
@@ -357,7 +373,6 @@ type_map = {
     'scrollback_pager': to_cmdline,
     'open_url_with': to_cmdline,
     'font_size': to_font_size,
-    'font_size_delta': positive_float,
     'focus_follows_mouse': to_bool,
     'cursor_shape': to_cursor_shape,
     'open_url_modifiers': to_modifiers,
diff --git a/kitty/core_text.m b/kitty/core_text.m
index b1246ac8c..2412d539a 100644
--- a/kitty/core_text.m
+++ b/kitty/core_text.m
@@ -303,7 +303,7 @@ face_from_path(const char *path, int UNUSED index, FONTS_DATA_HANDLE fg UNUSED)
 }
 
 PyObject*
-specialize_font_descriptor(PyObject *base_descriptor) {
+specialize_font_descriptor(PyObject *base_descriptor, FONTS_DATA_HANDLE fg UNUSED) {
     Py_INCREF(base_descriptor);
     return base_descriptor;
 }
diff --git a/kitty/fontconfig.c b/kitty/fontconfig.c
index 22d0d014b..d45bf4ae9 100644
--- a/kitty/fontconfig.c
+++ b/kitty/fontconfig.c
@@ -171,7 +171,7 @@ end:
 }
 
 PyObject*
-specialize_font_descriptor(PyObject *base_descriptor) {
+specialize_font_descriptor(PyObject *base_descriptor, FONTS_DATA_HANDLE fg) {
     PyObject *p = PyDict_GetItemString(base_descriptor, "path"), *ans = NULL;
     PyObject *idx = PyDict_GetItemString(base_descriptor, "index");
     if (p == NULL) { PyErr_SetString(PyExc_ValueError, "Base descriptor has no path"); return NULL; }
@@ -181,8 +181,8 @@ specialize_font_descriptor(PyObject *base_descriptor) {
     long face_idx = MAX(0, PyLong_AsLong(idx));
     AP(FcPatternAddString, FC_FILE, (const FcChar8*)PyUnicode_AsUTF8(p), "path");
     AP(FcPatternAddInteger, FC_INDEX, face_idx, "index");
-    AP(FcPatternAddDouble, FC_SIZE, global_state.font_sz_in_pts, "size");
-    AP(FcPatternAddDouble, FC_DPI, (global_state.default_dpi.x + global_state.default_dpi.y) / 2.0, "dpi");
+    AP(FcPatternAddDouble, FC_SIZE, fg->font_sz_in_pts, "size");
+    AP(FcPatternAddDouble, FC_DPI, (fg->logical_dpi_x + fg->logical_dpi_y) / 2.0, "dpi");
     ans = _fc_match(pat);
     if (face_idx > 0) {
         // For some reason FcFontMatch sets the index to zero, so manually restore it.
diff --git a/kitty/fonts.c b/kitty/fonts.c
index 88efbb030..79e8ef08e 100644
--- a/kitty/fonts.c
+++ b/kitty/fonts.c
@@ -336,7 +336,7 @@ sprite_tracker_set_layout(GPUSpriteTracker *sprite_tracker, unsigned int cell_wi
 
 static inline PyObject*
 desc_to_face(PyObject *desc, FONTS_DATA_HANDLE fg) {
-    PyObject *d = specialize_font_descriptor(desc);
+    PyObject *d = specialize_font_descriptor(desc, fg);
     if (d == NULL) return NULL;
     PyObject *ans = face_from_descriptor(d, fg);
     Py_DECREF(d);
diff --git a/kitty/fonts.h b/kitty/fonts.h
index b6f04dceb..380de0957 100644
--- a/kitty/fonts.h
+++ b/kitty/fonts.h
@@ -23,14 +23,13 @@ bool set_size_for_face(PyObject*, unsigned int, bool, FONTS_DATA_HANDLE);
 void cell_metrics(PyObject*, unsigned int*, unsigned int*, unsigned int*, unsigned int*, unsigned int*);
 bool render_glyphs_in_cells(PyObject *f, bool bold, bool italic, hb_glyph_info_t *info, hb_glyph_position_t *positions, unsigned int num_glyphs, pixel *canvas, unsigned int cell_width, unsigned int cell_height, unsigned int num_cells, unsigned int baseline, bool *was_colored, FONTS_DATA_HANDLE);
 PyObject* create_fallback_face(PyObject *base_face, Cell* cell, bool bold, bool italic, bool emoji_presentation, FONTS_DATA_HANDLE fg);
-PyObject* specialize_font_descriptor(PyObject *base_descriptor);
+PyObject* specialize_font_descriptor(PyObject *base_descriptor, FONTS_DATA_HANDLE);
 PyObject* face_from_path(const char *path, int index, FONTS_DATA_HANDLE);
 PyObject* face_from_descriptor(PyObject*, FONTS_DATA_HANDLE);
 
 void sprite_tracker_current_layout(FONTS_DATA_HANDLE data, unsigned int *x, unsigned int *y, unsigned int *z);
 void render_alpha_mask(uint8_t *alpha_mask, pixel* dest, Region *src_rect, Region *dest_rect, size_t src_stride, size_t dest_stride);
 void render_line(FONTS_DATA_HANDLE, Line *line);
-void load_fonts_for_window(OSWindow*);
 void sprite_tracker_set_limits(size_t max_texture_size, size_t max_array_len);
 typedef void (*free_extra_data_func)(void*);
 
diff --git a/kitty/freetype.c b/kitty/freetype.c
index 587bfbfa6..db5252160 100644
--- a/kitty/freetype.c
+++ b/kitty/freetype.c
@@ -130,10 +130,10 @@ set_font_size(Face *self, FT_F26Dot6 char_width, FT_F26Dot6 char_height, FT_UInt
 bool
 set_size_for_face(PyObject *s, unsigned int desired_height, bool force, FONTS_DATA_HANDLE fg) {
     Face *self = (Face*)s;
-    FT_F26Dot6 w = (FT_F26Dot6)(ceil(global_state.font_sz_in_pts * 64.0));
+    FT_F26Dot6 w = (FT_F26Dot6)(ceil(fg->font_sz_in_pts * 64.0));
     FT_UInt xdpi = (FT_UInt)fg->logical_dpi_x, ydpi = (FT_UInt)fg->logical_dpi_y;
     if (!force && (self->char_width == w && self->char_height == w && self->xdpi == xdpi && self->ydpi == ydpi)) return true;
-    ((Face*)self)->size_in_pts = global_state.font_sz_in_pts;
+    ((Face*)self)->size_in_pts = fg->font_sz_in_pts;
     return set_font_size(self, w, w, xdpi, ydpi, desired_height, fg->cell_height);
 }
 
diff --git a/kitty/graphics.c b/kitty/graphics.c
index 322654d18..596c957bc 100644
--- a/kitty/graphics.c
+++ b/kitty/graphics.c
@@ -810,7 +810,7 @@ grman_resize(GraphicsManager *self, index_type UNUSED old_lines, index_type UNUS
 }
 
 void
-grman_rescale(GraphicsManager *self, unsigned int UNUSED old_cell_width, unsigned int UNUSED old_cell_height, CellPixelSize cell) {
+grman_rescale(GraphicsManager *self, CellPixelSize cell) {
     ImageRef *ref; Image *img;
     self->layers_dirty = true;
     for (size_t i = self->image_count; i-- > 0;) {
diff --git a/kitty/graphics.h b/kitty/graphics.h
index 5fd657b7c..75db8db1c 100644
--- a/kitty/graphics.h
+++ b/kitty/graphics.h
@@ -88,4 +88,4 @@ const char* grman_handle_command(GraphicsManager *self, const GraphicsCommand *g
 bool grman_update_layers(GraphicsManager *self, unsigned int scrolled_by, float screen_left, float screen_top, float dx, float dy, unsigned int num_cols, unsigned int num_rows, CellPixelSize);
 void grman_scroll_images(GraphicsManager *self, const ScrollData*, CellPixelSize fg);
 void grman_resize(GraphicsManager*, index_type, index_type, index_type, index_type);
-void grman_rescale(GraphicsManager *self, unsigned int old_cell_width, unsigned int old_cell_height, CellPixelSize fg);
+void grman_rescale(GraphicsManager *self, CellPixelSize fg);
diff --git a/kitty/kitty.conf b/kitty/kitty.conf
index 2b693785b..2931edca4 100644
--- a/kitty/kitty.conf
+++ b/kitty/kitty.conf
@@ -28,11 +28,6 @@ bold_italic_font auto
 # Font size (in pts)
 font_size        11.0
 
-# The amount the font size is changed by (in pts) when increasing/decreasing
-# the font size in a running terminal.
-font_size_delta 2
-
-
 # Adjust the cell dimensions.
 # You can use either numbers, which are interpreted as pixels or percentages
 # (number followed by %), which are interpreted as percentages of the
@@ -461,12 +456,15 @@ map kitty_mod+alt+t    set_tab_title
 # }}}
 
 # Font sizes {{{
-map kitty_mod+equal    increase_font_size
-map kitty_mod+minus    decrease_font_size
-map kitty_mod+backspace restore_font_size
-# To setup shortcuts for specific font sizes, follow the example below:
-# map kitty_mod+f6     set_font_size 10.0
-# map kitty_mod+f7     set_font_size 20.5
+# You can change the font size for all top-level kitty windows at a time
+# or only the current one.
+map kitty_mod+equal    change_font_size all +2.0
+map kitty_mod+minus    change_font_size all -2.0
+map kitty_mod+backspace change_font_size all 0
+# To setup shortcuts for specific font sizes:
+# map kitty_mod+f6     change_font_size all 10.0
+# To setup shortcuts to change only the current window's font size:
+# map kitty_mod+f6     change_font_size current 10.0
 # }}}
 
 # Select and act on visible text {{{
diff --git a/kitty/screen.c b/kitty/screen.c
index 23fca7f40..917290641 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -130,6 +130,16 @@ screen_reset(Screen *self) {
     set_color_table_color(self, 104, NULL);
 }
 
+void
+screen_dirty_sprite_positions(Screen *self) {
+    self->is_dirty = true;
+    for (index_type i = 0; i < self->lines; i++) {
+        linebuf_mark_line_dirty(self->main_linebuf, i);
+        linebuf_mark_line_dirty(self->alt_linebuf, i);
+    }
+    for (index_type i = 0; i < self->historybuf->count; i++) historybuf_mark_line_dirty(self->historybuf, i);
+}
+
 static inline HistoryBuf*
 realloc_hb(HistoryBuf *old, unsigned int lines, unsigned int columns) {
     HistoryBuf *ans = alloc_historybuf(lines, columns);
@@ -206,10 +216,10 @@ screen_resize(Screen *self, unsigned int lines, unsigned int columns) {
     return true;
 }
 
-static void
-screen_rescale_images(Screen *self, unsigned int old_cell_width, unsigned int old_cell_height) {
-    grman_rescale(self->main_grman, old_cell_width, old_cell_height, self->cell_size);
-    grman_rescale(self->alt_grman, old_cell_width, old_cell_height, self->cell_size);
+void
+screen_rescale_images(Screen *self) {
+    grman_rescale(self->main_grman, self->cell_size);
+    grman_rescale(self->alt_grman, self->cell_size);
 }
 
 
@@ -1485,17 +1495,6 @@ as_text_non_visual(Screen *self, PyObject *args) {
     as_text_generic(args, self, range_line_, self->lines, self->columns);
 }
 
-static PyObject*
-refresh_sprite_positions(Screen *self, PyObject *a UNUSED) {
-    self->is_dirty = true;
-    for (index_type i = 0; i < self->lines; i++) {
-        linebuf_mark_line_dirty(self->main_linebuf, i);
-        linebuf_mark_line_dirty(self->alt_linebuf, i);
-    }
-    for (index_type i = 0; i < self->historybuf->count; i++) historybuf_mark_line_dirty(self->historybuf, i);
-    Py_RETURN_NONE;
-}
-
 static PyObject*
 screen_wcswidth(PyObject UNUSED *self, PyObject *str) {
     if (PyUnicode_READY(str) != 0) return NULL;
@@ -1692,7 +1691,7 @@ WRAP0(linefeed)
 WRAP0(carriage_return)
 WRAP2(resize, 1, 1)
 WRAP2(set_margins, 1, 1)
-WRAP2(rescale_images, 1, 1)
+WRAP0(rescale_images)
 WRAP2B(update_selection)
 
 static PyObject*
@@ -1980,7 +1979,6 @@ static PyMethodDef methods[] = {
     {"index", (PyCFunction)xxx_index, METH_VARARGS, ""},
     MND(as_text, METH_VARARGS)
     MND(as_text_non_visual, METH_VARARGS)
-    MND(refresh_sprite_positions, METH_NOARGS)
     MND(tab, METH_NOARGS)
     MND(backspace, METH_NOARGS)
     MND(linefeed, METH_NOARGS)
@@ -1993,7 +1991,7 @@ static PyMethodDef methods[] = {
     MND(mark_as_dirty, METH_NOARGS)
     MND(resize, METH_VARARGS)
     MND(set_margins, METH_VARARGS)
-    MND(rescale_images, METH_VARARGS)
+    MND(rescale_images, METH_NOARGS)
     MND(text_for_selection, METH_NOARGS)
     MND(scroll, METH_VARARGS)
     MND(send_escape_code_to_child, METH_VARARGS)
diff --git a/kitty/screen.h b/kitty/screen.h
index 3bc5b330b..e7e4d088c 100644
--- a/kitty/screen.h
+++ b/kitty/screen.h
@@ -170,6 +170,8 @@ unsigned long screen_current_char_width(Screen *self);
 void screen_mark_url(Screen *self, index_type start_x, index_type start_y, index_type end_x, index_type end_y);
 void screen_handle_graphics_command(Screen *self, const GraphicsCommand *cmd, const uint8_t *payload);
 bool screen_open_url(Screen*);
+void screen_dirty_sprite_positions(Screen *self);
+void screen_rescale_images(Screen *self);
 #define DECLARE_CH_SCREEN_HANDLER(name) void screen_##name(Screen *screen);
 DECLARE_CH_SCREEN_HANDLER(bell)
 DECLARE_CH_SCREEN_HANDLER(backspace)
diff --git a/kitty/state.c b/kitty/state.c
index db899458a..2a0e493aa 100644
--- a/kitty/state.c
+++ b/kitty/state.c
@@ -601,6 +601,45 @@ PYWRAP1(pt_to_px) {
     return PyLong_FromLong((long)round((pt * (dpi / 72.0))));
 }
 
+PYWRAP1(global_font_size) {
+    double set_val = -1;
+    PA("|d", &set_val);
+    if (set_val > 0) global_state.font_sz_in_pts = set_val;
+    return Py_BuildValue("d", global_state.font_sz_in_pts);
+}
+
+static inline void
+resize_screen(OSWindow *os_window, Screen *screen, bool has_graphics) {
+    if (screen) {
+        screen->cell_size.width = os_window->fonts_data->cell_width;
+        screen->cell_size.height = os_window->fonts_data->cell_height;
+        screen_dirty_sprite_positions(screen);
+        if (has_graphics) screen_rescale_images(screen);
+    }
+}
+
+PYWRAP1(os_window_font_size) {
+    id_type os_window_id;
+    int force = 0;
+    double new_sz = -1;
+    PA("K|dp", &os_window_id, &new_sz, &force);
+    WITH_OS_WINDOW(os_window_id)
+        if (new_sz > 0 && (force || new_sz != os_window->font_sz_in_pts)) {
+            os_window->font_sz_in_pts = new_sz;
+            load_fonts_for_window(os_window);
+            resize_screen(os_window, os_window->tab_bar_render_data.screen, false);
+            for (size_t ti = 0; ti < os_window->num_tabs; ti++) {
+                Tab *tab = os_window->tabs + ti;
+                for (size_t wi = 0; wi < tab->num_windows; wi++) {
+                    Window *w = tab->windows + wi;
+                    resize_screen(os_window, w->render_data.screen, true);
+                }
+            }
+        }
+        return Py_BuildValue("d", os_window->font_sz_in_pts);
+    END_WITH_OS_WINDOW
+    return Py_BuildValue("d", 0.0);
+}
 
 PYWRAP1(set_boss) {
     Py_CLEAR(global_state.boss);
@@ -659,6 +698,8 @@ static PyMethodDef module_methods[] = {
     MW(change_background_opacity, METH_VARARGS),
     MW(background_opacity_of, METH_O),
     MW(update_window_visibility, METH_VARARGS),
+    MW(global_font_size, METH_VARARGS),
+    MW(os_window_font_size, METH_VARARGS),
     MW(set_boss, METH_O),
     MW(destroy_global_data, METH_NOARGS),
 
diff --git a/kitty/state.h b/kitty/state.h
index f2f6b3d21..b08f6ba3d 100644
--- a/kitty/state.h
+++ b/kitty/state.h
@@ -186,3 +186,4 @@ void free_texture(uint32_t*);
 void send_image_to_gpu(uint32_t*, const void*, int32_t, int32_t, bool, bool);
 void send_sprite_to_gpu(FONTS_DATA_HANDLE fg, unsigned int, unsigned int, unsigned int, pixel*);
 void set_titlebar_color(OSWindow *w, color_type color);
+void load_fonts_for_window(OSWindow*);