feat: removed stencil from shadow rendering (#61)

* removed stencil from shadow rendering

* removed the rest of the stencil mask

* fix

* Don't render inner region of shadow

---------

Co-authored-by: Erik Reider <[email protected]>

7 files changed, 62 insertions, 133 deletions

diff --git a/render/fx_renderer/fx_pass.c b/render/fx_renderer/fx_pass.c
index 6c8a00a..dd45535 100644
--- a/render/fx_renderer/fx_pass.c
+++ b/render/fx_renderer/fx_pass.c
@@ -121,6 +121,40 @@ static const struct wlr_render_pass_impl render_pass_impl = {
 /// FX pass functions
 ///
 
+// TODO: REMOVE STENCILING
+
+// Initialize the stenciling work
+static void stencil_mask_init(void) {
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+	glEnable(GL_STENCIL_TEST);
+
+	glStencilFunc(GL_ALWAYS, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+	// Disable writing to color buffer
+	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+}
+
+// Close the mask
+static void stencil_mask_close(bool draw_inside_mask) {
+	// Reenable writing to color buffer
+	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	if (draw_inside_mask) {
+		glStencilFunc(GL_EQUAL, 1, 0xFF);
+		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+		return;
+	}
+	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+}
+
+// Finish stenciling and clear the buffer
+static void stencil_mask_fini(void) {
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+	glDisable(GL_STENCIL_TEST);
+}
+
 static void render(const struct wlr_box *box, const pixman_region32_t *clip, GLint attrib) {
 	pixman_region32_t region;
 	pixman_region32_init_rect(&region, box->x, box->y, box->width, box->height);
@@ -210,38 +244,6 @@ static void setup_blending(enum wlr_render_blend_mode mode) {
 	}
 }
 
-// Initialize the stenciling work
-static void stencil_mask_init(void) {
-	glClearStencil(0);
-	glClear(GL_STENCIL_BUFFER_BIT);
-	glEnable(GL_STENCIL_TEST);
-
-	glStencilFunc(GL_ALWAYS, 1, 0xFF);
-	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-	// Disable writing to color buffer
-	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-}
-
-// Close the mask
-static void stencil_mask_close(bool draw_inside_mask) {
-	// Reenable writing to color buffer
-	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
-	if (draw_inside_mask) {
-		glStencilFunc(GL_EQUAL, 1, 0xFF);
-		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-		return;
-	}
-	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
-	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
-}
-
-// Finish stenciling and clear the buffer
-static void stencil_mask_fini(void) {
-	glClearStencil(0);
-	glClear(GL_STENCIL_BUFFER_BIT);
-	glDisable(GL_STENCIL_TEST);
-}
-
 // make sure the texture source box does not try and sample outside of the
 // texture
 static void check_tex_src_box(const struct wlr_render_texture_options *options) {
@@ -324,7 +326,7 @@ void fx_render_pass_add_texture(struct fx_gles_render_pass *pass,
 
 	glUniform1i(shader->tex, 0);
 	glUniform1f(shader->alpha, alpha);
-	glUniform2f(shader->size, clip_box->width, clip_box->height);
+	glUniform2f(shader->half_size, (float)clip_box->width / 2.0, (float)clip_box->height / 2.0);
 	glUniform2f(shader->position, clip_box->x, clip_box->y);
 	glUniform1f(shader->radius, fx_options->corner_radius);
 	glUniform1f(shader->has_titlebar, fx_options->has_titlebar);
@@ -448,29 +450,6 @@ void fx_render_pass_add_rounded_border_corner(struct fx_gles_render_pass *pass,
 	pop_fx_debug(renderer);
 }
 
-void fx_render_pass_add_stencil_mask(struct fx_gles_render_pass *pass,
-		const struct fx_render_stencil_box_options *options) {
-
-	struct fx_renderer *renderer = pass->buffer->renderer;
-
-	struct wlr_box box = options->box;
-	assert(box.width > 0 && box.height > 0);
-
-	push_fx_debug(renderer);
-	setup_blending(WLR_RENDER_BLEND_MODE_PREMULTIPLIED);
-
-	glUseProgram(renderer->shaders.stencil_mask.program);
-
-	set_proj_matrix(renderer->shaders.stencil_mask.proj, pass->projection_matrix, &box);
-	glUniform2f(renderer->shaders.stencil_mask.half_size, box.width * 0.5, box.height * 0.5);
-	glUniform2f(renderer->shaders.stencil_mask.position, box.x, box.y);
-	glUniform1f(renderer->shaders.stencil_mask.radius, options->corner_radius);
-
-	render(&box, options->clip, renderer->shaders.stencil_mask.pos_attrib);
-
-	pop_fx_debug(renderer);
-}
-
 void fx_render_pass_add_box_shadow(struct fx_gles_render_pass *pass,
 		const struct fx_render_box_shadow_options *options) {
 	struct fx_renderer *renderer = pass->buffer->renderer;
@@ -479,8 +458,8 @@ void fx_render_pass_add_box_shadow(struct fx_gles_render_pass *pass,
 	const struct wlr_render_color *color = &shadow_data->color;
 	struct wlr_box shadow_box = options->shadow_box;
 	assert(shadow_box.width > 0 && shadow_box.height > 0);
+
 	struct wlr_box surface_box = options->clip_box;
-	float blur_sigma = shadow_data->blur_sigma;
 
 	pixman_region32_t render_region;
 	pixman_region32_init(&render_region);
@@ -496,17 +475,6 @@ void fx_render_pass_add_box_shadow(struct fx_gles_render_pass *pass,
 
 	push_fx_debug(renderer);
 
-	// Init stencil work
-	stencil_mask_init();
-	// Draw the rounded rect as a mask
-	struct fx_render_stencil_box_options stencil_options = {
-		.box = options->clip_box,
-		.corner_radius = options->corner_radius,
-		.clip = options->clip,
-	};
-	fx_render_pass_add_stencil_mask(pass, &stencil_options);
-	stencil_mask_close(false);
-
 	// blending will practically always be needed (unless we have a madman
 	// who uses opaque shadows with zero sigma), so just enable it
 	setup_blending(WLR_RENDER_BLEND_MODE_PREMULTIPLIED);
@@ -516,7 +484,7 @@ void fx_render_pass_add_box_shadow(struct fx_gles_render_pass *pass,
 
 	set_proj_matrix(renderer->shaders.box_shadow.proj, pass->projection_matrix, &shadow_box);
 	glUniform4f(renderer->shaders.box_shadow.color, color->r, color->g, color->b, color->a);
-	glUniform1f(renderer->shaders.box_shadow.blur_sigma, blur_sigma);
+	glUniform1f(renderer->shaders.box_shadow.blur_sigma, shadow_data->blur_sigma);
 	glUniform1f(renderer->shaders.box_shadow.corner_radius, options->corner_radius);
 	glUniform2f(renderer->shaders.box_shadow.size, shadow_box.width, shadow_box.height);
 	glUniform2f(renderer->shaders.box_shadow.position, shadow_box.x, shadow_box.y);
@@ -527,8 +495,6 @@ void fx_render_pass_add_box_shadow(struct fx_gles_render_pass *pass,
 
 	glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
 
-	stencil_mask_fini();
-
 	pop_fx_debug(renderer);
 }
 
diff --git a/render/fx_renderer/fx_renderer.c b/render/fx_renderer/fx_renderer.c
index d2d01db..d2ab366 100644
--- a/render/fx_renderer/fx_renderer.c
+++ b/render/fx_renderer/fx_renderer.c
@@ -238,7 +238,7 @@ static bool fx_render_subtexture_with_matrix(
 	glUniformMatrix3fv(shader->proj, 1, GL_FALSE, gl_matrix);
 	glUniform1i(shader->tex, 0);
 	glUniform1f(shader->alpha, alpha);
-	glUniform2f(shader->size, box->width, box->height);
+	glUniform2f(shader->half_size, box->width / 2.0, box->height / 2.0);
 	glUniform2f(shader->position, box->x, box->y);
 	glUniform1f(shader->radius, 0);
 	glUniform1f(shader->discard_transparent, false);
@@ -685,11 +685,6 @@ static bool link_shaders(struct fx_renderer *renderer) {
 		goto error;
 	}
 
-	// stencil mask shader
-	if (!link_stencil_mask_program(&renderer->shaders.stencil_mask)) {
-		wlr_log(WLR_ERROR, "Could not link stencil mask shader");
-		goto error;
-	}
 	// box shadow shader
 	if (!link_box_shadow_program(&renderer->shaders.box_shadow)) {
 		wlr_log(WLR_ERROR, "Could not link box shadow shader");
@@ -723,7 +718,6 @@ error:
 	glDeleteProgram(renderer->shaders.tex_rgbx.program);
 	glDeleteProgram(renderer->shaders.tex_ext.program);
 	glDeleteProgram(renderer->shaders.rounded_border_corner.program);
-	glDeleteProgram(renderer->shaders.stencil_mask.program);
 	glDeleteProgram(renderer->shaders.box_shadow.program);
 	glDeleteProgram(renderer->shaders.blur1.program);
 	glDeleteProgram(renderer->shaders.blur2.program);
diff --git a/render/fx_renderer/gles2/shaders/box_shadow.frag b/render/fx_renderer/gles2/shaders/box_shadow.frag
index 92d40fc..9ab9937 100644
--- a/render/fx_renderer/gles2/shaders/box_shadow.frag
+++ b/render/fx_renderer/gles2/shaders/box_shadow.frag
@@ -65,15 +65,22 @@ float random() {
     return fract(sin(dot(vec2(12.9898, 78.233), gl_FragCoord.xy)) * 43758.5453);
 }
 
+float roundRectSDF(vec2 half_size, vec2 position, float radius) {
+    vec2 q = abs(gl_FragCoord.xy - position - half_size) - half_size + radius;
+    return min(max(q.x, q.y), 0.0) + length(max(q, 0.0)) - radius;
+}
+
 void main() {
-    float frag_alpha = v_color.a * roundedBoxShadow(
+    float shadow_alpha = v_color.a * roundedBoxShadow(
             position + blur_sigma,
             position + size - blur_sigma,
             gl_FragCoord.xy, blur_sigma * 0.5,
             corner_radius);
-
     // dither the alpha to break up color bands
-    frag_alpha += (random() - 0.5) / 128.0;
+    shadow_alpha += (random() - 0.5) / 128.0;
+
+    // get the window alpha so we can render around the window
+    float window_alpha = 1.0 - smoothstep(-1.0, 1.0, roundRectSDF((size * 0.5) - blur_sigma, position + blur_sigma, corner_radius));
 
-    gl_FragColor = vec4(v_color.rgb, frag_alpha);
+    gl_FragColor = vec4(v_color.rgb, shadow_alpha * (1.0 - window_alpha));
 }
diff --git a/render/fx_renderer/gles2/shaders/meson.build b/render/fx_renderer/gles2/shaders/meson.build
index 427fcaf..f5b5b41 100644
--- a/render/fx_renderer/gles2/shaders/meson.build
+++ b/render/fx_renderer/gles2/shaders/meson.build
@@ -7,7 +7,6 @@ shaders = [
     'tex.frag',
     'rounded_border_corner.frag',
     'box_shadow.frag',
-    'stencil_mask.frag',
     'blur1.frag',
     'blur2.frag',
     'blur_effects.frag',
diff --git a/render/fx_renderer/gles2/shaders/stencil_mask.frag b/render/fx_renderer/gles2/shaders/stencil_mask.frag
deleted file mode 100644
index e1fd76a..0000000
--- a/render/fx_renderer/gles2/shaders/stencil_mask.frag
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifdef GL_FRAGMENT_PRECISION_HIGH
-precision highp float;
-#else
-precision mediump float;
-#endif
-
-varying vec2 v_texcoord;
-
-uniform vec2 half_size;
-uniform vec2 position;
-uniform float radius;
-
-void main() {
-	vec2 q = abs(gl_FragCoord.xy - position - half_size) - half_size + radius;
-	float dist = min(max(q.x,q.y), 0.0) + length(max(q, 0.0)) - radius;
-	float smoothedAlpha = 1.0 - smoothstep(-1.0, 0.5, dist);
-	gl_FragColor = mix(vec4(0.0), vec4(1.0), smoothedAlpha);
-
-	if (gl_FragColor.a < 1.0) {
-		discard;
-	}
-}
diff --git a/render/fx_renderer/gles2/shaders/tex.frag b/render/fx_renderer/gles2/shaders/tex.frag
index d08c95d..1377c00 100644
--- a/render/fx_renderer/gles2/shaders/tex.frag
+++ b/render/fx_renderer/gles2/shaders/tex.frag
@@ -26,7 +26,7 @@ uniform sampler2D tex;
 
 uniform float alpha;
 
-uniform vec2 size;
+uniform vec2 half_size;
 uniform vec2 position;
 uniform float radius;
 uniform bool has_titlebar;
@@ -42,19 +42,21 @@ vec4 sample_texture() {
 #endif
 }
 
+float roundRectSDF() {
+	vec2 q = abs(gl_FragCoord.xy - position - half_size) - half_size + radius;
+	return min(max(q.x, q.y), 0.0) + length(max(q, 0.0)) - radius;
+}
+
 void main() {
 	gl_FragColor = mix(sample_texture(), dim_color, dim) * alpha;
 
-	if (!has_titlebar || gl_FragCoord.y - position.y > radius) {
-		vec2 corner_distance = min(gl_FragCoord.xy - position, size + position - gl_FragCoord.xy);
-		if (max(corner_distance.x, corner_distance.y) < radius) {
-			float d = radius - distance(corner_distance, vec2(radius));
-			float smooth = smoothstep(-1.0, 0.5, d);
-			gl_FragColor = mix(vec4(0), gl_FragColor, smooth);
-		}
-	}
-
 	if (discard_transparent && gl_FragColor.a == 0.0) {
 		discard;
+		return;
+	}
+
+	if (!has_titlebar || gl_FragCoord.y - position.y > radius) {
+		float alpha = smoothstep(-1.0, 1.0, roundRectSDF());
+		gl_FragColor = mix(gl_FragColor, vec4(0.0), alpha);
 	}
 }
diff --git a/render/fx_renderer/shaders.c b/render/fx_renderer/shaders.c
index ad8fb3c..064136d 100644
--- a/render/fx_renderer/shaders.c
+++ b/render/fx_renderer/shaders.c
@@ -12,7 +12,6 @@
 #include "quad_round_frag_src.h"
 #include "tex_frag_src.h"
 #include "rounded_border_corner_frag_src.h"
-#include "stencil_mask_frag_src.h"
 #include "box_shadow_frag_src.h"
 #include "blur1_frag_src.h"
 #include "blur2_frag_src.h"
@@ -153,7 +152,7 @@ bool link_tex_program(struct tex_shader *shader,
 	shader->alpha = glGetUniformLocation(prog, "alpha");
 	shader->pos_attrib = glGetAttribLocation(prog, "pos");
 	shader->tex_proj = glGetUniformLocation(prog, "tex_proj");
-	shader->size = glGetUniformLocation(prog, "size");
+	shader->half_size = glGetUniformLocation(prog, "half_size");
 	shader->position = glGetUniformLocation(prog, "position");
 	shader->radius = glGetUniformLocation(prog, "radius");
 	shader->has_titlebar = glGetUniformLocation(prog, "has_titlebar");
@@ -186,22 +185,6 @@ bool link_rounded_border_corner_program(struct rounded_border_corner_shader *sha
 	return true;
 }
 
-bool link_stencil_mask_program(struct stencil_mask_shader *shader) {
-	GLuint prog;
-	shader->program = prog = link_program(stencil_mask_frag_src);
-	if (!shader->program) {
-		return false;
-	}
-
-	shader->proj = glGetUniformLocation(prog, "proj");
-	shader->pos_attrib = glGetAttribLocation(prog, "pos");
-	shader->position = glGetUniformLocation(prog, "position");
-	shader->half_size = glGetUniformLocation(prog, "half_size");
-	shader->radius = glGetUniformLocation(prog, "radius");
-
-	return true;
-}
-
 bool link_box_shadow_program(struct box_shadow_shader *shader) {
 	GLuint prog;
 	shader->program = prog = link_program(box_shadow_frag_src);