diff --git a/types/scene/wlr_scene.c b/types/scene/wlr_scene.c
index 7e5b3136..3a97cdaa 100644
--- a/types/scene/wlr_scene.c
+++ b/types/scene/wlr_scene.c
@@ -667,21 +667,21 @@ void wlr_scene_buffer_set_buffer_with_damage(struct wlr_scene_buffer *scene_buff
 	assert(buffer || !damage);
 
 	bool update = false;
-
-	wlr_texture_destroy(scene_buffer->texture);
-	scene_buffer->texture = NULL;
-
 	if (buffer) {
 		// if this node used to not be mapped or its previous displayed
 		// buffer region will be different from what the new buffer would
 		// produce we need to update the node.
-		update = scene_buffer->dst_width == 0 && scene_buffer->dst_height == 0 &&
-			(scene_buffer->buffer_width != buffer->width ||
-				scene_buffer->buffer_height != buffer->height);
+		update = (!scene_buffer->buffer && !scene_buffer->texture) ||
+			(scene_buffer->dst_width == 0 && scene_buffer->dst_height == 0 &&
+				(scene_buffer->buffer_width != buffer->width ||
+					scene_buffer->buffer_height != buffer->height));
 	} else {
 		update = true;
 	}
 
+	wlr_texture_destroy(scene_buffer->texture);
+	scene_buffer->texture = NULL;
+
 	scene_buffer_set_buffer(scene_buffer, buffer);
 
 	if (update) {