Fix: rotated outputs (#112)

4 files changed, 151 insertions, 39 deletions

diff --git a/include/sway/desktop/fx_renderer/fx_renderer.h b/include/sway/desktop/fx_renderer/fx_renderer.h
index 200ecb7b..109351e2 100644
--- a/include/sway/desktop/fx_renderer/fx_renderer.h
+++ b/include/sway/desktop/fx_renderer/fx_renderer.h
@@ -9,7 +9,7 @@
 #include "sway/desktop/fx_renderer/fx_framebuffer.h"
 #include "sway/desktop/fx_renderer/fx_texture.h"
 
-enum corner_location { ALL, TOP_LEFT, TOP_RIGHT, BOTTOM_LEFT, BOTTOM_RIGHT, NONE };
+enum corner_location { TOP_LEFT, TOP_RIGHT, BOTTOM_RIGHT, BOTTOM_LEFT, ALL, NONE };
 
 enum fx_tex_shader_source {
 	SHADER_SOURCE_TEXTURE_RGBA = 1,
@@ -21,6 +21,8 @@ enum fx_rounded_quad_shader_source {
 	SHADER_SOURCE_QUAD_ROUND = 1,
 	SHADER_SOURCE_QUAD_ROUND_TOP_LEFT = 2,
 	SHADER_SOURCE_QUAD_ROUND_TOP_RIGHT = 3,
+	SHADER_SOURCE_QUAD_ROUND_BOTTOM_RIGHT = 4,
+	SHADER_SOURCE_QUAD_ROUND_BOTTOM_LEFT = 5,
 };
 
 struct decoration_data {
@@ -106,6 +108,8 @@ struct fx_renderer {
 		struct rounded_quad_shader rounded_quad;
 		struct rounded_quad_shader rounded_tl_quad;
 		struct rounded_quad_shader rounded_tr_quad;
+		struct rounded_quad_shader rounded_bl_quad;
+		struct rounded_quad_shader rounded_br_quad;
 
 		struct blur_shader blur1;
 		struct blur_shader blur2;
@@ -165,15 +169,16 @@ void fx_render_rect(struct fx_renderer *renderer, const struct wlr_box *box,
 		const float color[static 4], const float projection[static 9]);
 
 void fx_render_rounded_rect(struct fx_renderer *renderer, const struct wlr_box *box,
-		const float color[static 4], const float projection[static 9],
-		int radius, enum corner_location corner_location);
+		const float color[static 4], const float matrix[static 9], int radius,
+		enum corner_location corner_location);
 
 void fx_render_border_corner(struct fx_renderer *renderer, const struct wlr_box *box,
-		const float color[static 4], const float projection[static 9],
+		const float color[static 4], const float matrix[static 9],
 		enum corner_location corner_location, int radius, int border_thickness);
 
 void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *box,
-		const float color[static 4], const float projection[static 9], int radius, float blur_sigma);
+		const float color[static 4], const float matrix[static 9], int radius,
+		float blur_sigma);
 
 void fx_render_blur(struct fx_renderer *renderer, const float matrix[static 9],
 		struct fx_framebuffer **buffer, struct blur_shader *shader, const struct wlr_box *box,
diff --git a/sway/desktop/fx_renderer/fx_renderer.c b/sway/desktop/fx_renderer/fx_renderer.c
index 7eb95ec9..8de11874 100644
--- a/sway/desktop/fx_renderer/fx_renderer.c
+++ b/sway/desktop/fx_renderer/fx_renderer.c
@@ -257,6 +257,14 @@ struct fx_renderer *fx_renderer_create(struct wlr_egl *egl) {
 			SHADER_SOURCE_QUAD_ROUND_TOP_RIGHT)) {
 		goto error;
 	}
+	if (!link_rounded_quad_program(renderer, &renderer->shaders.rounded_bl_quad,
+			SHADER_SOURCE_QUAD_ROUND_BOTTOM_LEFT)) {
+		goto error;
+	}
+	if (!link_rounded_quad_program(renderer, &renderer->shaders.rounded_br_quad,
+			SHADER_SOURCE_QUAD_ROUND_BOTTOM_RIGHT)) {
+		goto error;
+	}
 
 	// Border corner shader
 	prog = link_program(corner_frag_src);
@@ -345,6 +353,8 @@ error:
 	glDeleteProgram(renderer->shaders.rounded_quad.program);
 	glDeleteProgram(renderer->shaders.rounded_tl_quad.program);
 	glDeleteProgram(renderer->shaders.rounded_tr_quad.program);
+	glDeleteProgram(renderer->shaders.rounded_bl_quad.program);
+	glDeleteProgram(renderer->shaders.rounded_br_quad.program);
 	glDeleteProgram(renderer->shaders.corner.program);
 	glDeleteProgram(renderer->shaders.box_shadow.program);
 	glDeleteProgram(renderer->shaders.blur1.program);
@@ -567,8 +577,8 @@ void fx_render_rect(struct fx_renderer *renderer, const struct wlr_box *box,
 }
 
 void fx_render_rounded_rect(struct fx_renderer *renderer, const struct wlr_box *box,
-		const float color[static 4], const float projection[static 9],
-		int radius, enum corner_location corner_location) {
+		const float color[static 4], const float matrix[static 9], int radius,
+		enum corner_location corner_location) {
 	if (box->width == 0 || box->height == 0) {
 		return;
 	}
@@ -586,14 +596,17 @@ void fx_render_rounded_rect(struct fx_renderer *renderer, const struct wlr_box *
 		case TOP_RIGHT:
 			shader = &renderer->shaders.rounded_tr_quad;
 			break;
+		case BOTTOM_LEFT:
+			shader = &renderer->shaders.rounded_bl_quad;
+			break;
+		case BOTTOM_RIGHT:
+			shader = &renderer->shaders.rounded_br_quad;
+			break;
 		default:
 			sway_log(SWAY_ERROR, "Invalid Corner Location. Aborting render");
 			abort();
 	}
 
-	float matrix[9];
-	wlr_matrix_project_box(matrix, box, WL_OUTPUT_TRANSFORM_NORMAL, 0, projection);
-
 	float gl_matrix[9];
 	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
 
@@ -625,14 +638,12 @@ void fx_render_rounded_rect(struct fx_renderer *renderer, const struct wlr_box *
 }
 
 void fx_render_border_corner(struct fx_renderer *renderer, const struct wlr_box *box,
-		const float color[static 4], const float projection[static 9],
+		const float color[static 4], const float matrix[static 9],
 		enum corner_location corner_location, int radius, int border_thickness) {
 	if (border_thickness == 0 || box->width == 0 || box->height == 0) {
 		return;
 	}
 	assert(box->width > 0 && box->height > 0);
-	float matrix[9];
-	wlr_matrix_project_box(matrix, box, WL_OUTPUT_TRANSFORM_NORMAL, 0, projection);
 
 	float gl_matrix[9];
 	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
@@ -675,14 +686,12 @@ void fx_render_border_corner(struct fx_renderer *renderer, const struct wlr_box
 
 // TODO: alpha input arg?
 void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *box,
-		const float color[static 4], const float projection[static 9],
-		int corner_radius, float blur_sigma) {
+		const float color[static 4], const float matrix [static 9], int corner_radius,
+		float blur_sigma) {
 	if (box->width == 0 || box->height == 0) {
 		return;
 	}
 	assert(box->width > 0 && box->height > 0);
-	float matrix[9];
-	wlr_matrix_project_box(matrix, box, WL_OUTPUT_TRANSFORM_NORMAL, 0, projection);
 
 	float gl_matrix[9];
 	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
@@ -711,7 +720,7 @@ void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *bo
 	// Disable writing to color buffer
 	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
 	// Draw the rounded rect as a mask
-	fx_render_rounded_rect(renderer, &inner_box, col, projection, corner_radius, ALL);
+	fx_render_rounded_rect(renderer, &inner_box, col, matrix, corner_radius, ALL);
 	// Close the mask
 	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
 	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
diff --git a/sway/desktop/fx_renderer/shaders/quad_round.frag b/sway/desktop/fx_renderer/shaders/quad_round.frag
index 4dcf0c53..4099e60a 100644
--- a/sway/desktop/fx_renderer/shaders/quad_round.frag
+++ b/sway/desktop/fx_renderer/shaders/quad_round.frag
@@ -1,6 +1,8 @@
 #define SOURCE_QUAD_ROUND 1
 #define SOURCE_QUAD_ROUND_TOP_LEFT 2
 #define SOURCE_QUAD_ROUND_TOP_RIGHT 3
+#define SOURCE_QUAD_ROUND_BOTTOM_RIGHT 4
+#define SOURCE_QUAD_ROUND_BOTTOM_LEFT 5
 
 #if !defined(SOURCE)
 #error "Missing shader preamble"
@@ -22,6 +24,10 @@ vec2 getCornerDist() {
     return abs(gl_FragCoord.xy - position - size) - size + radius;
 #elif SOURCE == SOURCE_QUAD_ROUND_TOP_RIGHT
     return abs(gl_FragCoord.xy - position - vec2(0, size.y)) - size + radius;
+#elif SOURCE == SOURCE_QUAD_ROUND_BOTTOM_RIGHT
+    return abs(gl_FragCoord.xy - position) - size + radius;
+#elif SOURCE == SOURCE_QUAD_ROUND_BOTTOM_LEFT
+    return abs(gl_FragCoord.xy - position - vec2(size.x, 0)) - size + radius;
 #endif
 }
 
diff --git a/sway/desktop/render.c b/sway/desktop/render.c
index 2787181c..944775d6 100644
--- a/sway/desktop/render.c
+++ b/sway/desktop/render.c
@@ -65,6 +65,43 @@ bool should_parameters_blur() {
 	return config->blur_params.radius > 0 && config->blur_params.num_passes > 0;
 }
 
+// TODO: contribute wlroots function to allow creating an fbox from a box?
+struct wlr_fbox wlr_fbox_from_wlr_box(struct wlr_box *box) {
+	return (struct wlr_fbox) {
+		.x = box->x,
+		.y = box->y,
+		.width = box->width,
+		.height = box->height,
+	};
+}
+
+// TODO: Remove this ugly abomination with a complete border rework...
+enum corner_location get_rotated_corner(enum corner_location corner_location,
+		enum wl_output_transform transform) {
+	if (corner_location == ALL || corner_location == NONE) {
+		return corner_location;
+	}
+	switch (transform) {
+		case WL_OUTPUT_TRANSFORM_NORMAL:
+			return corner_location;
+		case WL_OUTPUT_TRANSFORM_90:
+			return (corner_location + 1) % 4;
+		case WL_OUTPUT_TRANSFORM_180:
+			return (corner_location + 2) % 4;
+		case WL_OUTPUT_TRANSFORM_270:
+			return (corner_location + 3) % 4;
+		case WL_OUTPUT_TRANSFORM_FLIPPED:
+			return (corner_location + (1 - 2 * (corner_location % 2))) % 4;
+		case WL_OUTPUT_TRANSFORM_FLIPPED_90:
+			return (corner_location + (4 - 2 * (corner_location % 2))) % 4;
+		case WL_OUTPUT_TRANSFORM_FLIPPED_180:
+			return (corner_location + (3 - 2 * (corner_location % 2))) % 4;
+		case WL_OUTPUT_TRANSFORM_FLIPPED_270:
+			return (corner_location + (2 - 2 * (corner_location % 2))) % 4;
+	}
+	return corner_location;
+}
+
 /**
  * Apply scale to a width or height.
  *
@@ -149,16 +186,23 @@ static void render_texture(struct wlr_output *wlr_output,
 		goto damage_finish;
 	}
 
+	// ensure the box is updated as per the output orientation
+	struct wlr_box transformed_box;
+	int width, height;
+	wlr_output_transformed_resolution(wlr_output, &width, &height);
+	wlr_box_transform(&transformed_box, dst_box,
+			wlr_output_transform_invert(wlr_output->transform), width, height);
+
 	int nrects;
 	pixman_box32_t *rects = pixman_region32_rectangles(&damage, &nrects);
 	for (int i = 0; i < nrects; ++i) {
 		scissor_output(wlr_output, &rects[i]);
 		set_scale_filter(wlr_output, texture, output->scale_filter);
 		if (src_box != NULL) {
-			fx_render_subtexture_with_matrix(renderer, texture, src_box, dst_box,
+			fx_render_subtexture_with_matrix(renderer, texture, src_box, &transformed_box,
 					matrix, deco_data);
 		} else {
-			fx_render_texture_with_matrix(renderer, texture, dst_box, matrix, deco_data);
+			fx_render_texture_with_matrix(renderer, texture, &transformed_box, matrix, deco_data);
 		}
 	}
 
@@ -171,7 +215,6 @@ void render_blur_segments(struct fx_renderer *renderer,
 		const float matrix[static 9], pixman_region32_t* damage,
 		struct fx_framebuffer **buffer, struct blur_shader* shader,
 		const struct wlr_box *box, int blur_radius) {
-
 	if (*buffer == &renderer->effects_buffer) {
 		fx_framebuffer_bind(&renderer->effects_buffer_swapped);
 	} else {
@@ -196,14 +239,15 @@ void render_blur_segments(struct fx_renderer *renderer,
 	}
 }
 
-/** Blurs the main_buffer content and returns the blurred framebuffer */
+// Blurs the main_buffer content and returns the blurred framebuffer
 struct fx_framebuffer *get_main_buffer_blur(struct fx_renderer *renderer, struct sway_output *output,
 		pixman_region32_t *original_damage, const float box_matrix[static 9], const struct wlr_box *box) {
-	struct wlr_box monitor_box = get_monitor_box(output->wlr_output);
+	struct wlr_output *wlr_output = output->wlr_output;
+	struct wlr_box monitor_box = get_monitor_box(wlr_output);
 
-	const enum wl_output_transform transform = wlr_output_transform_invert(output->wlr_output->transform);
+	const enum wl_output_transform transform = wlr_output_transform_invert(wlr_output->transform);
 	float matrix[9];
-	wlr_matrix_project_box(matrix, &monitor_box, transform, 0, output->wlr_output->transform_matrix);
+	wlr_matrix_project_box(matrix, &monitor_box, transform, 0, wlr_output->transform_matrix);
 
 	float gl_matrix[9];
 	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
@@ -211,7 +255,7 @@ struct fx_framebuffer *get_main_buffer_blur(struct fx_renderer *renderer, struct
 	pixman_region32_t damage;
 	pixman_region32_init(&damage);
 	pixman_region32_copy(&damage, original_damage);
-	wlr_region_transform(&damage, &damage, wlr_output_transform_invert(output->wlr_output->transform),
+	wlr_region_transform(&damage, &damage, transform,
 			monitor_box.width, monitor_box.height);
 	wlr_region_expand(&damage, &damage, get_blur_size());
 
@@ -341,6 +385,7 @@ static void render_surface_iterator(struct sway_output *output,
 	}
 
 	struct wlr_box proj_box = *_box;
+
 	scale_box(&proj_box, wlr_output->scale);
 
 	float matrix[9];
@@ -376,9 +421,10 @@ static void render_surface_iterator(struct sway_output *output,
 		}
 
 		if (has_alpha) {
-			int width, height;
-			wlr_output_transformed_resolution(wlr_output, &width, &height);
-			struct wlr_fbox blur_src_box = { 0, 0, width, height };
+			struct wlr_box monitor_box = get_monitor_box(wlr_output);
+			wlr_box_transform(&monitor_box, &monitor_box,
+					wlr_output_transform_invert(wlr_output->transform), monitor_box.width, monitor_box.height);
+			struct wlr_fbox blur_src_box = wlr_fbox_from_wlr_box(&monitor_box);
 			bool is_floating = container_is_floating(view->container);
 			render_blur(!is_floating, output, output_damage, &blur_src_box, &dst_box, &opaque_region,
 					surface->current.width, surface->current.height, surface->current.scale, deco_data.corner_radius);
@@ -481,7 +527,8 @@ void render_monitor_blur(struct sway_output *output, pixman_region32_t *damage)
 			wlr_output->transform_matrix, &monitor_box);
 
 	// Render the newly blurred content into the blur_buffer
-	fx_framebuffer_create(&renderer->blur_buffer, monitor_box.width, monitor_box.height, true);
+	fx_framebuffer_create(&renderer->blur_buffer,
+			output->renderer->viewport_width, output->renderer->viewport_height, true);
 	// Clear the damaged region of the blur_buffer
 	float clear_color[] = { 0, 0, 0, 0 };
 	int nrects;
@@ -545,11 +592,25 @@ void render_rounded_rect(struct sway_output *output, pixman_region32_t *output_d
 		goto damage_finish;
 	}
 
+	float matrix[9];
+	wlr_matrix_project_box(matrix, &box, WL_OUTPUT_TRANSFORM_NORMAL, 0,
+			wlr_output->transform_matrix);
+
+	enum wl_output_transform transform = wlr_output_transform_invert(wlr_output->transform);
+
+	// ensure the box is updated as per the output orientation
+	struct wlr_box transformed_box;
+	int width, height;
+	wlr_output_transformed_resolution(wlr_output, &width, &height);
+	wlr_box_transform(&transformed_box, &box, transform, width, height);
+
+	corner_location = get_rotated_corner(corner_location, transform);
+
 	int nrects;
 	pixman_box32_t *rects = pixman_region32_rectangles(&damage, &nrects);
 	for (int i = 0; i < nrects; ++i) {
 		scissor_output(wlr_output, &rects[i]);
-		fx_render_rounded_rect(renderer, &box, color, wlr_output->transform_matrix,
+		fx_render_rounded_rect(renderer, &transformed_box, color, matrix,
 				corner_radius, corner_location);
 	}
 
@@ -576,11 +637,25 @@ void render_border_corner(struct sway_output *output, pixman_region32_t *output_
 		goto damage_finish;
 	}
 
+	float matrix[9];
+	wlr_matrix_project_box(matrix, &box, WL_OUTPUT_TRANSFORM_NORMAL, 0,
+			wlr_output->transform_matrix);
+
+	enum wl_output_transform transform = wlr_output_transform_invert(wlr_output->transform);
+
+	// ensure the box is updated as per the output orientation
+	struct wlr_box transformed_box;
+	int width, height;
+	wlr_output_transformed_resolution(wlr_output, &width, &height);
+	wlr_box_transform(&transformed_box, &box, transform, width, height);
+
+	corner_location = get_rotated_corner(corner_location, transform);
+
 	int nrects;
 	pixman_box32_t *rects = pixman_region32_rectangles(&damage, &nrects);
 	for (int i = 0; i < nrects; ++i) {
 		scissor_output(wlr_output, &rects[i]);
-		fx_render_border_corner(renderer, &box, color, wlr_output->transform_matrix,
+		fx_render_border_corner(renderer, &transformed_box, color, matrix,
 				corner_location, corner_radius, border_thickness);
 	}
 
@@ -620,13 +695,24 @@ void render_box_shadow(struct sway_output *output, pixman_region32_t *output_dam
 		goto damage_finish;
 	}
 
+	float matrix[9];
+	wlr_matrix_project_box(matrix, &box, WL_OUTPUT_TRANSFORM_NORMAL, 0,
+			wlr_output->transform_matrix);
+
+	// ensure the box is updated as per the output orientation
+	struct wlr_box transformed_box;
+	int width, height;
+	wlr_output_transformed_resolution(wlr_output, &width, &height);
+	wlr_box_transform(&transformed_box, &box,
+			wlr_output_transform_invert(wlr_output->transform), width, height);
+
 	int nrects;
 	pixman_box32_t *rects = pixman_region32_rectangles(&damage, &nrects);
 	for (int i = 0; i < nrects; ++i) {
 		scissor_output(wlr_output, &rects[i]);
 
-		fx_render_box_shadow(renderer, &box, color,
-				wlr_output->transform_matrix, corner_radius, blur_sigma);
+		fx_render_box_shadow(renderer, &transformed_box, color, matrix,
+				corner_radius, blur_sigma);
 	}
 
 damage_finish:
@@ -740,8 +826,9 @@ static void render_saved_view(struct sway_view *view, struct sway_output *output
 				pixman_region32_union_rect(&opaque_region, &opaque_region, 0, 0, 0, 0);
 
 				struct wlr_box monitor_box = get_monitor_box(wlr_output);
-				// TODO: contribute wlroots function to allow creating an fbox from a box?
-				struct wlr_fbox src_box = { monitor_box.x, monitor_box.y, monitor_box.width, monitor_box.height };
+				wlr_box_transform(&monitor_box, &monitor_box,
+						wlr_output_transform_invert(wlr_output->transform), monitor_box.width, monitor_box.height);
+				struct wlr_fbox src_box = wlr_fbox_from_wlr_box(&monitor_box);
 				bool is_floating = container_is_floating(view->container);
 				render_blur(!is_floating, output, damage, &src_box, &dst_box, &opaque_region,
 						saved_buf->width, saved_buf->height, 1, deco_data.corner_radius);
@@ -1757,11 +1844,16 @@ void output_render(struct sway_output *output, struct timespec *when,
 		fullscreen_con = workspace->current.fullscreen;
 	}
 
-	int width, height;
-	wlr_output_transformed_resolution(wlr_output, &width, &height);
 
-	fx_renderer_begin(renderer, width, height);
+	struct wlr_box monitor_box = get_monitor_box(wlr_output);
+	wlr_box_transform(&monitor_box, &monitor_box,
+			wlr_output_transform_invert(wlr_output->transform),
+			monitor_box.width, monitor_box.height);
+
+	fx_renderer_begin(renderer, monitor_box.width, monitor_box.height);
 
+	int width, height;
+	wlr_output_transformed_resolution(wlr_output, &width, &height);
 	if (debug.damage == DAMAGE_RERENDER) {
 		pixman_region32_union_rect(damage, damage, 0, 0, width, height);
 	}