feat: add box shadows (#16)

6 files changed, 331 insertions, 1 deletions

diff --git a/render/fx_renderer/fx_renderer.c b/render/fx_renderer/fx_renderer.c
index 014be33..730e314 100644
--- a/render/fx_renderer/fx_renderer.c
+++ b/render/fx_renderer/fx_renderer.c
@@ -15,12 +15,15 @@
 #include <wlr/util/log.h>
 
 #include "render/fx_renderer/fx_renderer.h"
+#include "render/fx_renderer/fx_stencilbuffer.h"
 #include "render/fx_renderer/matrix.h"
 
 // shaders
 #include "common_vert_src.h"
 #include "quad_frag_src.h"
 #include "tex_frag_src.h"
+#include "stencil_mask_frag_src.h"
+#include "box_shadow_frag_src.h"
 
 static const GLfloat verts[] = {
 	1, 0, // top right
@@ -120,6 +123,40 @@ static bool link_tex_program(struct tex_shader *shader,
 	return true;
 }
 
+static bool link_stencil_mask_program(struct stencil_mask_shader *shader) {
+	GLuint prog;
+	shader->program = prog = link_program(stencil_mask_frag_src);
+	if (!shader->program) {
+		return false;
+	}
+
+	shader->proj = glGetUniformLocation(prog, "proj");
+	shader->color = glGetUniformLocation(prog, "color");
+	shader->pos_attrib = glGetAttribLocation(prog, "pos");
+	shader->position = glGetUniformLocation(prog, "position");
+	shader->half_size = glGetUniformLocation(prog, "half_size");
+	shader->radius = glGetUniformLocation(prog, "radius");
+
+	return true;
+}
+
+static bool link_box_shadow_program(struct box_shadow_shader *shader) {
+	GLuint prog;
+	shader->program = prog = link_program(box_shadow_frag_src);
+	if (!shader->program) {
+		return false;
+	}
+	shader->proj = glGetUniformLocation(prog, "proj");
+	shader->color = glGetUniformLocation(prog, "color");
+	shader->pos_attrib = glGetAttribLocation(prog, "pos");
+	shader->position = glGetUniformLocation(prog, "position");
+	shader->size = glGetUniformLocation(prog, "size");
+	shader->blur_sigma = glGetUniformLocation(prog, "blur_sigma");
+	shader->corner_radius = glGetUniformLocation(prog, "corner_radius");
+
+	return true;
+}
+
 static bool check_gl_ext(const char *exts, const char *ext) {
 	size_t extlen = strlen(ext);
 	const char *end = exts + strlen(exts);
@@ -191,6 +228,8 @@ struct fx_renderer *fx_renderer_create(struct wlr_egl *egl) {
 		return NULL;
 	}
 
+	renderer->stencil_buffer = fx_stencilbuffer_create();
+
 	// get extensions
 	const char *exts_str = (const char *)glGetString(GL_EXTENSIONS);
 	if (exts_str == NULL) {
@@ -226,6 +265,15 @@ struct fx_renderer *fx_renderer_create(struct wlr_egl *egl) {
 		goto error;
 	}
 
+	// stencil mask shader
+	if (!link_stencil_mask_program(&renderer->shaders.stencil_mask)) {
+		goto error;
+	}
+	// box shadow shader
+	if (!link_box_shadow_program(&renderer->shaders.box_shadow)) {
+		goto error;
+	}
+
 	if (!eglMakeCurrent(wlr_egl_get_display(egl),
 				EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT)) {
 		wlr_log(WLR_ERROR, "GLES2 RENDERER: Could not unset current EGL");
@@ -240,6 +288,8 @@ error:
 	glDeleteProgram(renderer->shaders.tex_rgba.program);
 	glDeleteProgram(renderer->shaders.tex_rgbx.program);
 	glDeleteProgram(renderer->shaders.tex_ext.program);
+	glDeleteProgram(renderer->shaders.stencil_mask.program);
+	glDeleteProgram(renderer->shaders.box_shadow.program);
 
 	if (!eglMakeCurrent(wlr_egl_get_display(egl),
 				EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT)) {
@@ -254,10 +304,12 @@ error:
 }
 
 void fx_renderer_fini(struct fx_renderer *renderer) {
-	// NO OP
+	fx_stencilbuffer_release(&renderer->stencil_buffer);
 }
 
 void fx_renderer_begin(struct fx_renderer *renderer, int width, int height) {
+	fx_stencilbuffer_init(&renderer->stencil_buffer, width, height);
+
 	glViewport(0, 0, width, height);
 
 	// refresh projection matrix
@@ -282,6 +334,43 @@ void fx_renderer_scissor(struct wlr_box *box) {
 	}
 }
 
+void fx_renderer_stencil_mask_init(void) {
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+	glEnable(GL_STENCIL_TEST);
+
+	glStencilFunc(GL_ALWAYS, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+	// Disable writing to color buffer
+	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+}
+
+void fx_renderer_stencil_mask_close(bool draw_inside_mask) {
+	// Reenable writing to color buffer
+	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	if (draw_inside_mask) {
+		glStencilFunc(GL_EQUAL, 1, 0xFF);
+		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+		return;
+	}
+	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+}
+
+void fx_renderer_stencil_mask_fini(void) {
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+	glDisable(GL_STENCIL_TEST);
+}
+
+void fx_renderer_stencil_enable(void) {
+	glEnable(GL_STENCIL_TEST);
+}
+
+void fx_renderer_stencil_disable(void) {
+	glDisable(GL_STENCIL_TEST);
+}
+
 bool fx_render_subtexture_with_matrix(struct fx_renderer *renderer,
 		struct wlr_texture *wlr_texture, const struct wlr_fbox *src_box,
 		const struct wlr_box *dst_box, const float matrix[static 9],
@@ -410,3 +499,100 @@ void fx_render_rect(struct fx_renderer *renderer, const struct wlr_box *box,
 
 	glDisableVertexAttribArray(shader.pos_attrib);
 }
+
+static void fx_render_stencil_mask(struct fx_renderer *renderer,
+		const struct wlr_box *box, const float matrix[static 9],
+		int corner_radius) {
+	if (box->width == 0 || box->height == 0) {
+		return;
+	}
+	assert(box->width > 0 && box->height > 0);
+
+	// TODO: just pass gl_matrix?
+	float gl_matrix[9];
+	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
+
+	// TODO: investigate why matrix is flipped prior to this cmd
+	// wlr_matrix_multiply(gl_matrix, flip_180, gl_matrix);
+
+	wlr_matrix_transpose(gl_matrix, gl_matrix);
+
+	glEnable(GL_BLEND);
+
+	struct stencil_mask_shader shader = renderer->shaders.stencil_mask;
+
+	glUseProgram(shader.program);
+
+	glUniformMatrix3fv(shader.proj, 1, GL_FALSE, gl_matrix);
+
+	glUniform2f(shader.half_size, box->width * 0.5, box->height * 0.5);
+	glUniform2f(shader.position, box->x, box->y);
+	glUniform1f(shader.radius, corner_radius);
+
+	glVertexAttribPointer(shader.pos_attrib, 2, GL_FLOAT, GL_FALSE,
+			0, verts);
+
+	glEnableVertexAttribArray(shader.pos_attrib);
+
+	glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+	glDisableVertexAttribArray(shader.pos_attrib);
+}
+
+void fx_render_box_shadow(struct fx_renderer *renderer,
+		const struct wlr_box *box, const struct wlr_box *stencil_box,
+		const float matrix[static 9], int corner_radius,
+		struct shadow_data *shadow_data) {
+	if (box->width == 0 || box->height == 0) {
+		return;
+	}
+	assert(box->width > 0 && box->height > 0);
+
+	float *color = shadow_data->color;
+	float blur_sigma = shadow_data->blur_sigma;
+
+	float gl_matrix[9];
+	wlr_matrix_multiply(gl_matrix, renderer->projection, matrix);
+
+	// TODO: investigate why matrix is flipped prior to this cmd
+	// wlr_matrix_multiply(gl_matrix, flip_180, gl_matrix);
+
+	wlr_matrix_transpose(gl_matrix, gl_matrix);
+
+	// Init stencil work
+	fx_renderer_stencil_mask_init();
+	// Draw the rounded rect as a mask
+	fx_render_stencil_mask(renderer, stencil_box, matrix, corner_radius);
+	fx_renderer_stencil_mask_close(false);
+
+	// blending will practically always be needed (unless we have a madman
+	// who uses opaque shadows with zero sigma), so just enable it
+	glEnable(GL_BLEND);
+
+	glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+
+	struct box_shadow_shader shader = renderer->shaders.box_shadow;
+
+	glUseProgram(shader.program);
+
+	glUniformMatrix3fv(shader.proj, 1, GL_FALSE, gl_matrix);
+	glUniform4f(shader.color, color[0], color[1], color[2], color[3]);
+	glUniform1f(shader.blur_sigma, blur_sigma);
+	glUniform1f(shader.corner_radius, corner_radius);
+
+	glUniform2f(shader.size, box->width, box->height);
+	glUniform2f(shader.position, box->x, box->y);
+
+	glVertexAttribPointer(shader.pos_attrib, 2, GL_FLOAT, GL_FALSE,
+			0, verts);
+
+	glEnableVertexAttribArray(shader.pos_attrib);
+
+	glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+	glDisableVertexAttribArray(shader.pos_attrib);
+
+	glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
+
+	fx_renderer_stencil_mask_fini();
+}
diff --git a/render/fx_renderer/fx_stencilbuffer.c b/render/fx_renderer/fx_stencilbuffer.c
new file mode 100644
index 0000000..4f57216
--- /dev/null
+++ b/render/fx_renderer/fx_stencilbuffer.c
@@ -0,0 +1,50 @@
+#include <assert.h>
+#include <wlr/render/gles2.h>
+#include <wlr/util/log.h>
+
+#include "include/render/fx_renderer/fx_stencilbuffer.h"
+
+struct fx_stencilbuffer fx_stencilbuffer_create(void) {
+	return (struct fx_stencilbuffer) {
+		.rb = -1,
+		.width = -1,
+		.height = -1,
+	};
+}
+
+void fx_stencilbuffer_init(struct fx_stencilbuffer *stencil_buffer, int width, int height) {
+	bool first_alloc = false;
+
+	if (stencil_buffer->rb == (uint32_t) -1) {
+		glGenRenderbuffers(1, &stencil_buffer->rb);
+		first_alloc = true;
+	}
+
+	if (first_alloc || stencil_buffer->width != width || stencil_buffer->height != height) {
+		glBindRenderbuffer(GL_RENDERBUFFER, stencil_buffer->rb);
+		glRenderbufferStorage(GL_RENDERBUFFER, GL_STENCIL_INDEX8, width, height);
+		stencil_buffer->width = width;
+		stencil_buffer->height = height;
+
+		GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
+		if (status != GL_FRAMEBUFFER_COMPLETE) {
+			wlr_log(WLR_ERROR,
+					"Stencil buffer incomplete, couldn't create! (FB status: %i)",
+					status);
+			return;
+		}
+	}
+
+	// Reattach the RenderBuffer to the FrameBuffer
+	glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT,
+			GL_RENDERBUFFER, stencil_buffer->rb);
+}
+
+void fx_stencilbuffer_release(struct fx_stencilbuffer *stencil_buffer) {
+	if (stencil_buffer->rb != (uint32_t) -1 && stencil_buffer->rb) {
+		glDeleteRenderbuffers(1, &stencil_buffer->rb);
+	}
+	stencil_buffer->rb = -1;
+	stencil_buffer->width = -1;
+	stencil_buffer->height = -1;
+}
diff --git a/render/fx_renderer/gles2/shaders/box_shadow.frag b/render/fx_renderer/gles2/shaders/box_shadow.frag
new file mode 100644
index 0000000..c9b2b91
--- /dev/null
+++ b/render/fx_renderer/gles2/shaders/box_shadow.frag
@@ -0,0 +1,74 @@
+// Writeup: https://madebyevan.com/shaders/fast-rounded-rectangle-shadows/
+
+precision mediump float;
+varying vec4 v_color;
+varying vec2 v_texcoord;
+
+uniform vec2 position;
+uniform vec2 size;
+uniform float blur_sigma;
+uniform float corner_radius;
+
+float gaussian(float x, float sigma) {
+    const float pi = 3.141592653589793;
+    return exp(-(x * x) / (2.0 * sigma * sigma)) / (sqrt(2.0 * pi) * sigma);
+}
+
+// approximates the error function, needed for the gaussian integral
+vec2 erf(vec2 x) {
+    vec2 s = sign(x), a = abs(x);
+    x = 1.0 + (0.278393 + (0.230389 + 0.078108 * (a * a)) * a) * a;
+    x *= x;
+    return s - s / (x * x);
+}
+
+// return the blurred mask along the x dimension
+float roundedBoxShadowX(float x, float y, float sigma, float corner, vec2 halfSize) {
+    float delta = min(halfSize.y - corner - abs(y), 0.0);
+    float curved = halfSize.x - corner + sqrt(max(0.0, corner * corner - delta * delta));
+    vec2 integral = 0.5 + 0.5 * erf((x + vec2(-curved, curved)) * (sqrt(0.5) / sigma));
+    return integral.y - integral.x;
+}
+
+// return the mask for the shadow of a box from lower to upper
+float roundedBoxShadow(vec2 lower, vec2 upper, vec2 point, float sigma, float corner_radius) {
+    // Center everything to make the math easier
+    vec2 center = (lower + upper) * 0.5;
+    vec2 halfSize = (upper - lower) * 0.5;
+    point -= center;
+
+    // The signal is only non-zero in a limited range, so don't waste samples
+    float low = point.y - halfSize.y;
+    float high = point.y + halfSize.y;
+    float start = clamp(-3.0 * sigma, low, high);
+    float end = clamp(3.0 * sigma, low, high);
+
+    // Accumulate samples (we can get away with surprisingly few samples)
+    float step = (end - start) / 4.0;
+    float y = start + step * 0.5;
+    float value = 0.0;
+    for (int i = 0; i < 4; i++) {
+        value += roundedBoxShadowX(point.x, point.y - y, sigma, corner_radius, halfSize) * gaussian(y, sigma) * step;
+        y += step;
+    }
+
+    return value;
+}
+
+// per-pixel "random" number between 0 and 1
+float random() {
+    return fract(sin(dot(vec2(12.9898, 78.233), gl_FragCoord.xy)) * 43758.5453);
+}
+
+void main() {
+    float frag_alpha = v_color.a * roundedBoxShadow(
+            position + blur_sigma,
+            position + size - blur_sigma,
+            gl_FragCoord.xy, blur_sigma * 0.5,
+            corner_radius);
+
+    // dither the alpha to break up color bands
+    frag_alpha += (random() - 0.5) / 128.0;
+
+    gl_FragColor = vec4(v_color.rgb, frag_alpha);
+}
diff --git a/render/fx_renderer/gles2/shaders/meson.build b/render/fx_renderer/gles2/shaders/meson.build
index ee030a2..42f066e 100644
--- a/render/fx_renderer/gles2/shaders/meson.build
+++ b/render/fx_renderer/gles2/shaders/meson.build
@@ -4,6 +4,8 @@ shaders = [
     'common.vert',
     'quad.frag',
     'tex.frag',
+    'box_shadow.frag',
+    'stencil_mask.frag',
 ]
 
 foreach name : shaders
diff --git a/render/fx_renderer/gles2/shaders/stencil_mask.frag b/render/fx_renderer/gles2/shaders/stencil_mask.frag
new file mode 100644
index 0000000..ee03307
--- /dev/null
+++ b/render/fx_renderer/gles2/shaders/stencil_mask.frag
@@ -0,0 +1,17 @@
+precision mediump float;
+varying vec2 v_texcoord;
+
+uniform vec2 half_size;
+uniform vec2 position;
+uniform float radius;
+
+void main() {
+    vec2 q = abs(gl_FragCoord.xy - position - half_size) - half_size + radius;
+    float dist = min(max(q.x,q.y), 0.0) + length(max(q, 0.0)) - radius;
+    float smoothedAlpha = 1.0 - smoothstep(-1.0, 0.5, dist);
+    gl_FragColor = mix(vec4(0.0), vec4(1.0), smoothedAlpha);
+
+    if (gl_FragColor.a < 1.0) {
+        discard;
+    }
+}
diff --git a/render/fx_renderer/meson.build b/render/fx_renderer/meson.build
index b4c13a0..394caa3 100644
--- a/render/fx_renderer/meson.build
+++ b/render/fx_renderer/meson.build
@@ -7,6 +7,7 @@ endif
 
 wlr_files += files(
 	'matrix.c',
+	'fx_stencilbuffer.c',
 	'fx_renderer.c',
 )