diff --git a/kitty/graphics.c b/kitty/graphics.c
index f06f5cfe2..966bd29c5 100644
--- a/kitty/graphics.c
+++ b/kitty/graphics.c
@@ -620,6 +620,31 @@ grman_update_layers(GraphicsManager *self, unsigned int scrolled_by, float scree
 
 // }}}
 
+// Image lifetime/scrolling {{{
+
+void 
+grman_scroll_images(GraphicsManager *self, int32_t amt, int32_t limit) {
+    Image *img; ImageRef *ref;
+    size_t i, j;
+
+    if (self->image_count) {
+        for (i = 0; i < self->image_count; i++) { img = self->images + i; for (j = 0; j < img->refcnt; j++) { ref = img->refs + j;
+            ref->start_row += amt;
+            if (ref->start_row < limit) {
+                // TODO: remove refs that have scrolled beyond limit
+            }
+        }}
+        self->layers_dirty = true;
+    }
+}
+
+void
+grman_clear(GraphicsManager UNUSED *self) {
+    // TODO: Implement this
+}
+
+// }}}
+
 const char*
 grman_handle_command(GraphicsManager *self, const GraphicsCommand *g, const uint8_t *payload, Cursor *c, bool *is_dirty) {
     Image *image;
@@ -647,11 +672,6 @@ grman_handle_command(GraphicsManager *self, const GraphicsCommand *g, const uint
     return ret;
 }
 
-void
-grman_clear(GraphicsManager UNUSED *self) {
-    // TODO: Implement this
-}
-
 
 // Boilerplate {{{
 static PyObject *
diff --git a/kitty/graphics.h b/kitty/graphics.h
index 4b573590c..8653c9c2b 100644
--- a/kitty/graphics.h
+++ b/kitty/graphics.h
@@ -79,3 +79,4 @@ GraphicsManager* grman_realloc(GraphicsManager *, index_type lines, index_type c
 void grman_clear(GraphicsManager*);
 const char* grman_handle_command(GraphicsManager *self, const GraphicsCommand *g, const uint8_t *payload, Cursor *c, bool *is_dirty);
 bool grman_update_layers(GraphicsManager *self, unsigned int scrolled_by, float screen_left, float screen_top, float dx, float dy, unsigned int num_cols, unsigned int num_rows);
+void grman_scroll_images(GraphicsManager *self, int32_t amt, int32_t limit);
diff --git a/kitty/screen.c b/kitty/screen.c
index e15915834..e35bc10f9 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -654,6 +654,7 @@ screen_cursor_to_column(Screen *self, unsigned int column) {
 
 #define INDEX_UP \
     linebuf_index(self->linebuf, top, bottom); \
+    grman_scroll_images(self->grman, -1, self->linebuf == self->main_linebuf ? -self->historybuf->ynum : 0); \
     if (self->linebuf == self->main_linebuf && bottom == self->lines - 1) { \
         /* Only add to history when no page margins have been set */ \
         linebuf_init_line(self->linebuf, bottom); \