Add layer effect option to ignore transparent regions when blurring (#159)

11 files changed, 124 insertions, 36 deletions

diff --git a/README.md b/README.md
index 086dc006..f69e2cb2 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,7 @@ Sway is an incredible window manager, and certainly one of the most well establi
     - SwayIPC Example: `swaymsg "layer_effects 'waybar' 'blur enable; shadows enable; corner_radius 6'"`
     - Available Effects:
         - `blur <enable|disable>`
+        - `blur_ignore_transparent <enable|disable>`
         - `shadows <enable|disable>`
         - `corner_radius <int>`
 + Dim unfocused windows:
diff --git a/include/sway/desktop/fx_renderer/fx_renderer.h b/include/sway/desktop/fx_renderer/fx_renderer.h
index 67dc6adf..410e3d94 100644
--- a/include/sway/desktop/fx_renderer/fx_renderer.h
+++ b/include/sway/desktop/fx_renderer/fx_renderer.h
@@ -32,6 +32,7 @@ struct decoration_data {
 	float dim;
 	float *dim_color;
 	bool has_titlebar;
+	bool discard_transparent;
 	bool blur;
 	bool shadow;
 };
@@ -113,6 +114,7 @@ struct tex_shader {
 	GLint dim;
 	GLint dim_color;
 	GLint has_titlebar;
+	GLint discard_transparent;
 };
 
 struct fx_renderer {
@@ -177,6 +179,15 @@ void fx_renderer_clear(const float color[static 4]);
 
 void fx_renderer_scissor(struct wlr_box *box);
 
+// Initialize the stenciling work
+void fx_renderer_stencil_mask_init();
+
+// Close the mask
+void fx_renderer_stencil_mask_close(bool draw_inside_mask);
+
+// Finish stenciling and clear the buffer
+void fx_renderer_stencil_mask_fini();
+
 bool fx_render_subtexture_with_matrix(struct fx_renderer *renderer, struct fx_texture *fx_texture,
 		const struct wlr_fbox *src_box, const struct wlr_box *dst_box, const float matrix[static 9],
 		struct decoration_data deco_data);
diff --git a/include/sway/layers.h b/include/sway/layers.h
index b04990dc..5871f0d8 100644
--- a/include/sway/layers.h
+++ b/include/sway/layers.h
@@ -30,6 +30,7 @@ struct sway_layer_surface {
 
 	bool has_shadow;
 	bool has_blur;
+	bool blur_ignore_transparent;
 	int corner_radius;
 };
 
diff --git a/sway/desktop/fx_renderer/fx_renderer.c b/sway/desktop/fx_renderer/fx_renderer.c
index c0a39716..d2fef6f1 100644
--- a/sway/desktop/fx_renderer/fx_renderer.c
+++ b/sway/desktop/fx_renderer/fx_renderer.c
@@ -222,6 +222,7 @@ static bool link_tex_program(struct tex_shader *shader,
 	shader->radius = glGetUniformLocation(prog, "radius");
 	shader->saturation = glGetUniformLocation(prog, "saturation");
 	shader->has_titlebar = glGetUniformLocation(prog, "has_titlebar");
+	shader->discard_transparent = glGetUniformLocation(prog, "discard_transparent");
 
 	return true;
 }
@@ -448,6 +449,35 @@ void fx_renderer_scissor(struct wlr_box *box) {
 	}
 }
 
+void fx_renderer_stencil_mask_init() {
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+	glEnable(GL_STENCIL_TEST);
+
+	glStencilFunc(GL_ALWAYS, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+	// Disable writing to color buffer
+	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+}
+
+void fx_renderer_stencil_mask_close(bool draw_inside_mask) {
+	// Reenable writing to color buffer
+	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	if (draw_inside_mask) {
+		glStencilFunc(GL_EQUAL, 1, 0xFF);
+		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+		return;
+	}
+	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+}
+
+void fx_renderer_stencil_mask_fini() {
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+	glDisable(GL_STENCIL_TEST);
+}
+
 bool fx_render_subtexture_with_matrix(struct fx_renderer *renderer, struct fx_texture *fx_texture,
 		const struct wlr_fbox *src_box, const struct wlr_box *dst_box, const float matrix[static 9],
 		struct decoration_data deco_data) {
@@ -509,6 +539,7 @@ bool fx_render_subtexture_with_matrix(struct fx_renderer *renderer, struct fx_te
 	glUniform1f(shader->dim, deco_data.dim);
 	glUniform4f(shader->dim_color, dim_color[0], dim_color[1], dim_color[2], dim_color[3]);
 	glUniform1f(shader->has_titlebar, deco_data.has_titlebar);
+	glUniform1f(shader->discard_transparent, deco_data.discard_transparent);
 	glUniform1f(shader->saturation, deco_data.saturation);
 	glUniform1f(shader->radius, deco_data.corner_radius);
 
@@ -737,7 +768,6 @@ void fx_render_stencil_mask(struct fx_renderer *renderer, const struct wlr_box *
 	glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
 
 	glDisableVertexAttribArray(shader.pos_attrib);
-
 }
 
 // TODO: alpha input arg?
@@ -765,21 +795,10 @@ void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *bo
 	inner_box.width -= blur_sigma * 2;
 	inner_box.height -= blur_sigma * 2;
 
-	glEnable(GL_STENCIL_TEST);
-	glClearStencil(0);
-	glClear(GL_STENCIL_BUFFER_BIT);
-
-	glStencilFunc(GL_ALWAYS, 1, 0xFF);
-	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-	// Disable writing to color buffer
-	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+	fx_renderer_stencil_mask_init();
 	// Draw the rounded rect as a mask
 	fx_render_stencil_mask(renderer, &inner_box, matrix, corner_radius);
-	// Close the mask
-	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
-	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
-	// Reenable writing to color buffer
-	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	fx_renderer_stencil_mask_close(false);
 
 	// blending will practically always be needed (unless we have a madman
 	// who uses opaque shadows with zero sigma), so just enable it
@@ -810,9 +829,7 @@ void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *bo
 
 	glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
 
-	glClearStencil(0);
-	glClear(GL_STENCIL_BUFFER_BIT);
-	glDisable(GL_STENCIL_TEST);
+	fx_renderer_stencil_mask_fini();
 }
 
 void fx_render_blur(struct fx_renderer *renderer, const float matrix[static 9],
@@ -854,4 +871,5 @@ void fx_render_blur(struct fx_renderer *renderer, const float matrix[static 9],
 
 	glDisableVertexAttribArray(shader->pos_attrib);
 	glDisableVertexAttribArray(shader->tex_attrib);
+
 }
diff --git a/sway/desktop/fx_renderer/shaders/tex.frag b/sway/desktop/fx_renderer/shaders/tex.frag
index 817b838c..77501887 100644
--- a/sway/desktop/fx_renderer/shaders/tex.frag
+++ b/sway/desktop/fx_renderer/shaders/tex.frag
@@ -26,8 +26,9 @@ uniform vec4 dim_color;
 uniform vec2 size;
 uniform vec2 position;
 uniform float radius;
-uniform bool has_titlebar;
 uniform float saturation;
+uniform bool has_titlebar;
+uniform bool discard_transparent;
 
 const vec3 saturation_weight = vec3(0.2125, 0.7154, 0.0721);
 
@@ -59,4 +60,8 @@ void main() {
             gl_FragColor = mix(vec4(0), gl_FragColor, smooth);
         }
     }
+
+    if (discard_transparent && gl_FragColor.a == 0.0) {
+        discard;
+    }
 }
diff --git a/sway/desktop/layer_shell.c b/sway/desktop/layer_shell.c
index 8a828dc6..d0d56755 100644
--- a/sway/desktop/layer_shell.c
+++ b/sway/desktop/layer_shell.c
@@ -711,6 +711,7 @@ void handle_layer_shell_surface(struct wl_listener *listener, void *data) {
 	layer_surface->data = sway_layer;
 
 	sway_layer->has_blur = false;
+	sway_layer->blur_ignore_transparent = false;
 	sway_layer->has_shadow = false;
 	sway_layer->corner_radius = 0;
 
diff --git a/sway/desktop/output.c b/sway/desktop/output.c
index 911bffbb..c41088ac 100644
--- a/sway/desktop/output.c
+++ b/sway/desktop/output.c
@@ -203,6 +203,7 @@ void output_layer_for_each_toplevel_surface(struct sway_output *output,
 		struct render_data *data = user_data;
 		data->deco_data.blur = layer_surface->layer != ZWLR_LAYER_SHELL_V1_LAYER_BACKGROUND ?
 			layer_surface->has_blur : false;
+		data->deco_data.discard_transparent = layer_surface->blur_ignore_transparent;
 		data->deco_data.shadow = layer_surface->has_shadow;
 		data->deco_data.corner_radius = layer_surface->corner_radius;
 
diff --git a/sway/desktop/render.c b/sway/desktop/render.c
index d97c3609..f5f697c4 100644
--- a/sway/desktop/render.c
+++ b/sway/desktop/render.c
@@ -40,6 +40,7 @@ struct decoration_data get_undecorated_decoration_data() {
 		.saturation = 1.0f,
 		.has_titlebar = false,
 		.blur = false,
+		.discard_transparent = false,
 		.shadow = false,
 	};
 }
@@ -266,9 +267,16 @@ struct fx_framebuffer *get_main_buffer_blur(struct fx_renderer *renderer, struct
 	return current_buffer;
 }
 
-void render_blur(bool optimized, struct sway_output *output, pixman_region32_t *output_damage,
-		const struct wlr_box *dst_box, pixman_region32_t *opaque_region, int corner_radius,
-		bool should_round_top) {
+struct blur_stencil_data {
+	struct fx_texture *stencil_texture;
+	const struct wlr_fbox *stencil_src_box;
+	float *stencil_matrix;
+};
+
+void render_blur(bool optimized, struct sway_output *output,
+		pixman_region32_t *output_damage, const struct wlr_box *dst_box,
+		pixman_region32_t *opaque_region, struct decoration_data *deco_data,
+		struct blur_stencil_data *stencil_data) {
 	struct wlr_output *wlr_output = output->wlr_output;
 	struct fx_renderer *renderer = output->renderer;
 
@@ -299,16 +307,32 @@ void render_blur(bool optimized, struct sway_output *output, pixman_region32_t *
 		buffer = get_main_buffer_blur(renderer, output, &translucent_region, dst_box);
 	}
 
+	// Get a stencil of the window ignoring transparent regions
+	if (deco_data->discard_transparent) {
+		fx_renderer_scissor(NULL);
+		fx_renderer_stencil_mask_init();
+
+		render_texture(wlr_output, output_damage, stencil_data->stencil_texture, stencil_data->stencil_src_box,
+				dst_box, stencil_data->stencil_matrix, *deco_data);
+
+		fx_renderer_stencil_mask_close(true);
+	}
+
 	// Draw the blurred texture
 	struct wlr_box monitor_box = get_monitor_box(wlr_output);
 	enum wl_output_transform transform = wlr_output_transform_invert(wlr_output->transform);
 	float matrix[9];
 	wlr_matrix_project_box(matrix, &monitor_box, transform, 0.0, wlr_output->transform_matrix);
 
-	struct decoration_data deco_data = get_undecorated_decoration_data();
-	deco_data.corner_radius = corner_radius;
-	deco_data.has_titlebar = should_round_top;
-	render_texture(wlr_output, &damage, &buffer->texture, NULL, dst_box, matrix, deco_data);
+	struct decoration_data blur_deco_data = get_undecorated_decoration_data();
+	blur_deco_data.corner_radius = deco_data->corner_radius;
+	blur_deco_data.has_titlebar = deco_data->has_titlebar;
+	render_texture(wlr_output, &damage, &buffer->texture, NULL, dst_box, matrix, blur_deco_data);
+
+	// Finish stenciling
+	if (deco_data->discard_transparent) {
+		fx_renderer_stencil_mask_fini();
+	}
 
 damage_finish:
 	pixman_region32_fini(&damage);
@@ -404,6 +428,10 @@ static void render_surface_iterator(struct sway_output *output,
 	struct decoration_data deco_data = data->deco_data;
 	deco_data.corner_radius *= wlr_output->scale;
 
+	struct wlr_fbox src_box;
+	wlr_surface_get_buffer_source_box(surface, &src_box);
+	struct fx_texture fx_texture = fx_texture_from_wlr_texture(texture);
+
 	// render blur
 	bool is_subsurface = view ? view->surface != surface : false;
 	if (deco_data.blur && config_should_parameters_blur() && !is_subsurface) {
@@ -420,19 +448,21 @@ static void render_surface_iterator(struct sway_output *output,
 		}
 
 		if (has_alpha) {
-			bool should_optimize_blur = view ? !container_is_floating_or_child(view->container) ||
-				config->blur_xray : false;
-			render_blur(should_optimize_blur, output, output_damage, &dst_box, &opaque_region,
-					deco_data.corner_radius, deco_data.has_titlebar);
+			struct wlr_box monitor_box = get_monitor_box(wlr_output);
+			wlr_box_transform(&monitor_box, &monitor_box,
+					wlr_output_transform_invert(wlr_output->transform), monitor_box.width, monitor_box.height);
+			struct blur_stencil_data stencil_data = { &fx_texture, &src_box, matrix };
+			bool should_optimize_blur = view ? !container_is_floating(view->container) || config->blur_xray : false;
+			render_blur(should_optimize_blur, output, output_damage, &dst_box,
+					&opaque_region, &deco_data, &stencil_data);
 		}
 
 		pixman_region32_fini(&opaque_region);
 	}
 
+	deco_data.discard_transparent = false;
+
 	// Render surface texture
-	struct wlr_fbox src_box;
-	wlr_surface_get_buffer_source_box(surface, &src_box);
-	struct fx_texture fx_texture = fx_texture_from_wlr_texture(texture);
 	render_texture(wlr_output, output_damage, &fx_texture, &src_box, &dst_box,
 		matrix, deco_data);
 
@@ -448,7 +478,7 @@ static void render_layer_iterator(struct sway_output *output,
 	struct decoration_data deco_data = data->deco_data;
 
 	// Ignore effects if this is a subsurface
-	if (wl_list_length(&surface->current.subsurfaces_above) > 0) {
+	if (!wlr_surface_is_layer_surface(surface)) {
 		deco_data = get_undecorated_decoration_data();
 	}
 
@@ -786,6 +816,8 @@ static void render_saved_view(struct sway_view *view, struct sway_output *output
 
 		deco_data.corner_radius *= wlr_output->scale;
 
+		struct fx_texture fx_texture = fx_texture_from_wlr_texture(saved_buf->buffer->texture);
+
 		// render blur
 		if (deco_data.blur && config_should_parameters_blur()) {
 			struct wlr_gles2_texture_attribs attribs;
@@ -796,16 +828,21 @@ static void render_saved_view(struct sway_view *view, struct sway_output *output
 				pixman_region32_init(&opaque_region);
 				pixman_region32_union_rect(&opaque_region, &opaque_region, 0, 0, 0, 0);
 
-				bool should_optimize_blur = !container_is_floating_or_child(view->container) || config->blur_xray;
+				struct wlr_box monitor_box = get_monitor_box(wlr_output);
+				wlr_box_transform(&monitor_box, &monitor_box,
+						wlr_output_transform_invert(wlr_output->transform), monitor_box.width, monitor_box.height);
+				struct blur_stencil_data stencil_data = { &fx_texture, &saved_buf->source_box, matrix };
+				bool should_optimize_blur = !container_is_floating(view->container) || config->blur_xray;
 				render_blur(should_optimize_blur, output, damage, &dst_box, &opaque_region,
-						deco_data.corner_radius, deco_data.has_titlebar);
+						&deco_data, &stencil_data);
 
 				pixman_region32_fini(&opaque_region);
 			}
 		}
 
+		deco_data.discard_transparent = false;
+
 		// Render saved surface texture
-		struct fx_texture fx_texture = fx_texture_from_wlr_texture(saved_buf->buffer->texture);
 		render_texture(wlr_output, damage, &fx_texture,
 				&saved_buf->source_box, &dst_box, matrix, deco_data);
 	}
@@ -1405,6 +1442,7 @@ static void render_containers_linear(struct sway_output *output,
 				.saturation = child->saturation,
 				.has_titlebar = has_titlebar,
 				.blur = child->blur_enabled,
+				.discard_transparent = false,
 				.shadow = child->shadow_enabled,
 			};
 			render_view(output, damage, child, colors, deco_data);
@@ -1455,6 +1493,7 @@ static void render_containers_tabbed(struct sway_output *output,
 		.saturation = current->saturation,
 		.has_titlebar = true,
 		.blur = current->blur_enabled,
+		.discard_transparent = false,
 		.shadow = current->shadow_enabled,
 	};
 
@@ -1551,6 +1590,7 @@ static void render_containers_stacked(struct sway_output *output,
 				? 0 : current->corner_radius,
 		.has_titlebar = true,
 		.blur = current->blur_enabled,
+		.discard_transparent = false,
 		.shadow = current->shadow_enabled,
 	};
 
@@ -1699,6 +1739,7 @@ static void render_floating_container(struct sway_output *soutput,
 			.corner_radius = con->corner_radius,
 			.has_titlebar = has_titlebar,
 			.blur = con->blur_enabled,
+			.discard_transparent = false,
 			.shadow = con->shadow_enabled,
 		};
 		render_view(soutput, damage, con, colors, deco_data);
@@ -1956,6 +1997,7 @@ void output_render(struct sway_output *output, struct timespec *when,
 			.saturation = focus->saturation,
 			.has_titlebar = false,
 			.blur = false,
+			.discard_transparent = false,
 			.shadow = false,
 		};
 		render_view_popups(focus->view, output, damage, deco_data);
diff --git a/sway/ipc-json.c b/sway/ipc-json.c
index 673a941a..2ae09457 100644
--- a/sway/ipc-json.c
+++ b/sway/ipc-json.c
@@ -377,6 +377,10 @@ static void ipc_json_describe_enabled_output(struct sway_output *output,
 			if (lsurface->has_blur) {
 				json_object_array_add(effects, json_object_new_string("blur"));
 			}
+			if (lsurface->blur_ignore_transparent) {
+				json_object_array_add(effects,
+						json_object_new_string("blur_ignore_transparent"));
+			}
 			if (lsurface->has_shadow) {
 				json_object_array_add(effects, json_object_new_string("shadows"));
 			}
diff --git a/sway/layer_criteria.c b/sway/layer_criteria.c
index 694c5016..f7a1b084 100644
--- a/sway/layer_criteria.c
+++ b/sway/layer_criteria.c
@@ -67,6 +67,9 @@ void layer_criteria_parse(struct sway_layer_surface *sway_layer, struct layer_cr
 		if (strcmp(argv[0], "blur") == 0) {
 			sway_layer->has_blur = parse_boolean(argv[1], true);
 			continue;
+		} if (strcmp(argv[0], "blur_ignore_transparent") == 0) {
+			sway_layer->blur_ignore_transparent = parse_boolean(argv[1], true);
+			continue;
 		} else if (strcmp(argv[0], "shadows") == 0) {
 			sway_layer->has_shadow = parse_boolean(argv[1], true);
 			continue;
diff --git a/sway/sway.5.scd b/sway/sway.5.scd
index 06c27900..a68eee5c 100644
--- a/sway/sway.5.scd
+++ b/sway/sway.5.scd
@@ -795,6 +795,7 @@ The default colors are:
 
 	Effects:
 	- *blur* <enable|disable>
+	- *blur_ignore_transparent* <enable|disable>
 	- *shadows* <enable|disable>
 	- *corner_radius* <integer>