Remove fmt parameter from wlr_texture_write_pixels

It's not allowed to change the format of a texture so remove
the confusing parameter.

include/render/gles2.h

@@ -72,6 +72,7 @@ struct wlr_gles2_texture {
 	enum wlr_gles2_texture_type type;
 	int width, height;
 	bool has_alpha;
+	uint32_t wl_format; // used to interpret upload data
 	bool inverted_y;
 
 	// Not set if WLR_GLES2_TEXTURE_GLTEX

include/wlr/render/interface.h

@@ -66,9 +66,9 @@ struct wlr_texture_impl {
 	void (*get_size)(struct wlr_texture *texture, int *width, int *height);
 	bool (*is_opaque)(struct wlr_texture *texture);
 	bool (*write_pixels)(struct wlr_texture *texture,
-		enum wl_shm_format wl_fmt, uint32_t stride, uint32_t width,
+		uint32_t stride, uint32_t width, uint32_t height,
-		uint32_t height, uint32_t src_x, uint32_t src_y, uint32_t dst_x,
+		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
-		uint32_t dst_y, const void *data);
+		const void *data);
 	bool (*to_dmabuf)(struct wlr_texture *texture,
 		struct wlr_dmabuf_attributes *attribs);
 	void (*destroy)(struct wlr_texture *texture);

include/wlr/render/wlr_texture.h

@@ -55,9 +55,11 @@ bool wlr_texture_is_opaque(struct wlr_texture *texture);
 
 /**
  * Update a texture with raw pixels. The texture must be mutable.
+ * The given data is interpreted as being in the format the
+ * texture was created with.
  */
 bool wlr_texture_write_pixels(struct wlr_texture *texture,
-	enum wl_shm_format wl_fmt, uint32_t stride, uint32_t width, uint32_t height,
+	uint32_t stride, uint32_t width, uint32_t height,
 	uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
 	const void *data);
 

render/gles2/texture.c

@@ -44,9 +44,9 @@ static bool gles2_texture_is_opaque(struct wlr_texture *wlr_texture) {
 }
 
 static bool gles2_texture_write_pixels(struct wlr_texture *wlr_texture,
-		enum wl_shm_format wl_fmt, uint32_t stride, uint32_t width,
+		uint32_t stride, uint32_t width, uint32_t height,
-		uint32_t height, uint32_t src_x, uint32_t src_y, uint32_t dst_x,
+		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
-		uint32_t dst_y, const void *data) {
+		const void *data) {
 	struct wlr_gles2_texture *texture =
 		get_gles2_texture_in_context(wlr_texture);
 
@@ -55,11 +55,9 @@ static bool gles2_texture_write_pixels(struct wlr_texture *wlr_texture,
 		return false;
 	}
 
-	const struct wlr_gles2_pixel_format *fmt = get_gles2_format_from_wl(wl_fmt);
+	const struct wlr_gles2_pixel_format *fmt = get_gles2_format_from_wl(
-	if (fmt == NULL) {
+		texture->wl_format);
-		wlr_log(WLR_ERROR, "Unsupported pixel format %"PRIu32, wl_fmt);
+	assert(fmt);
-		return false;
-	}
 
 	// TODO: what if the unpack subimage extension isn't supported?
 	PUSH_GLES2_DEBUG;
@@ -167,6 +165,7 @@ struct wlr_texture *wlr_gles2_texture_from_pixels(struct wlr_egl *egl,
 	texture->height = height;
 	texture->type = WLR_GLES2_TEXTURE_GLTEX;
 	texture->has_alpha = fmt->has_alpha;
+	texture->wl_format = fmt->wl_format;
 
 	PUSH_GLES2_DEBUG;
 
@@ -203,6 +202,7 @@ struct wlr_texture *wlr_gles2_texture_from_wl_drm(struct wlr_egl *egl,
 	texture->wl_drm = data;
 
 	EGLint fmt;
+	texture->wl_format = 0xFFFFFFFF; // texture can't be written anyways
 	texture->image = wlr_egl_create_image_from_wl_drm(egl, data, &fmt,
 		&texture->width, &texture->height, &texture->inverted_y);
 	if (texture->image == NULL) {
@@ -283,6 +283,7 @@ struct wlr_texture *wlr_gles2_texture_from_dmabuf(struct wlr_egl *egl,
 	texture->height = attribs->height;
 	texture->type = WLR_GLES2_TEXTURE_DMABUF;
 	texture->has_alpha = true;
+	texture->wl_format = 0xFFFFFFFF; // texture can't be written anyways
 	texture->inverted_y =
 		(attribs->flags & WLR_DMABUF_ATTRIBUTES_FLAGS_Y_INVERT) != 0;
 

render/wlr_texture.c

@@ -55,10 +55,10 @@ bool wlr_texture_is_opaque(struct wlr_texture *texture) {
 }
 
 bool wlr_texture_write_pixels(struct wlr_texture *texture,
-		enum wl_shm_format wl_fmt, uint32_t stride, uint32_t width,
+		uint32_t stride, uint32_t width, uint32_t height,
-		uint32_t height, uint32_t src_x, uint32_t src_y, uint32_t dst_x,
+		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
-		uint32_t dst_y, const void *data) {
+		const void *data) {
-	return texture->impl->write_pixels(texture, wl_fmt, stride, width, height,
+	return texture->impl->write_pixels(texture, stride, width, height,
 		src_x, src_y, dst_x, dst_y, data);
 }
 

types/wlr_buffer.c

@@ -150,12 +150,20 @@ struct wlr_buffer *wlr_buffer_apply_damage(struct wlr_buffer *buffer,
 	}
 
 	struct wl_shm_buffer *shm_buf = wl_shm_buffer_get(resource);
-	if (shm_buf == NULL) {
+	struct wl_shm_buffer *old_shm_buf = wl_shm_buffer_get(buffer->resource);
-		// Uploading only damaged regions only works for wl_shm buffers
+	if (shm_buf == NULL || old_shm_buf == NULL) {
+		// Uploading only damaged regions only works for wl_shm buffers and
+		// mutable textures (created from wl_shm buffer)
+		return NULL;
+	}
+
+	enum wl_shm_format new_fmt = wl_shm_buffer_get_format(shm_buf);
+	enum wl_shm_format old_fmt = wl_shm_buffer_get_format(old_shm_buf);
+	if (new_fmt != old_fmt) {
+		// Uploading to textures can't change the format
 		return NULL;
 	}
 
-	enum wl_shm_format fmt = wl_shm_buffer_get_format(shm_buf);
 	int32_t stride = wl_shm_buffer_get_stride(shm_buf);
 	int32_t width = wl_shm_buffer_get_width(shm_buf);
 	int32_t height = wl_shm_buffer_get_height(shm_buf);
@@ -173,7 +181,7 @@ struct wlr_buffer *wlr_buffer_apply_damage(struct wlr_buffer *buffer,
 	pixman_box32_t *rects = pixman_region32_rectangles(damage, &n);
 	for (int i = 0; i < n; ++i) {
 		pixman_box32_t *r = &rects[i];
-		if (!wlr_texture_write_pixels(buffer->texture, fmt, stride,
+		if (!wlr_texture_write_pixels(buffer->texture, stride,
 				r->x2 - r->x1, r->y2 - r->y1, r->x1, r->y1,
 				r->x1, r->y1, data)) {
 			wl_shm_buffer_end_access(shm_buf);