4 files changed, 23 insertions, 37 deletions
diff --git a/render/fx_renderer/gles2/shaders/box_shadow.frag b/render/fx_renderer/gles2/shaders/box_shadow.frag
index 92d40fc..9ab9937 100644
--- a/render/fx_renderer/gles2/shaders/box_shadow.frag
+++ b/render/fx_renderer/gles2/shaders/box_shadow.frag
@@ -65,15 +65,22 @@ float random() {
     return fract(sin(dot(vec2(12.9898, 78.233), gl_FragCoord.xy)) * 43758.5453);
 }
 
+float roundRectSDF(vec2 half_size, vec2 position, float radius) {
+    vec2 q = abs(gl_FragCoord.xy - position - half_size) - half_size + radius;
+    return min(max(q.x, q.y), 0.0) + length(max(q, 0.0)) - radius;
+}
+
 void main() {
-    float frag_alpha = v_color.a * roundedBoxShadow(
+    float shadow_alpha = v_color.a * roundedBoxShadow(
             position + blur_sigma,
             position + size - blur_sigma,
             gl_FragCoord.xy, blur_sigma * 0.5,
             corner_radius);
-
     // dither the alpha to break up color bands
-    frag_alpha += (random() - 0.5) / 128.0;
+    shadow_alpha += (random() - 0.5) / 128.0;
+
+    // get the window alpha so we can render around the window
+    float window_alpha = 1.0 - smoothstep(-1.0, 1.0, roundRectSDF((size * 0.5) - blur_sigma, position + blur_sigma, corner_radius));
 
-    gl_FragColor = vec4(v_color.rgb, frag_alpha);
+    gl_FragColor = vec4(v_color.rgb, shadow_alpha * (1.0 - window_alpha));
 }
diff --git a/render/fx_renderer/gles2/shaders/meson.build b/render/fx_renderer/gles2/shaders/meson.build
index 427fcaf..f5b5b41 100644
--- a/render/fx_renderer/gles2/shaders/meson.build
+++ b/render/fx_renderer/gles2/shaders/meson.build
@@ -7,7 +7,6 @@ shaders = [
     'tex.frag',
     'rounded_border_corner.frag',
     'box_shadow.frag',
-    'stencil_mask.frag',
     'blur1.frag',
     'blur2.frag',
     'blur_effects.frag',
diff --git a/render/fx_renderer/gles2/shaders/stencil_mask.frag b/render/fx_renderer/gles2/shaders/stencil_mask.frag
deleted file mode 100644
index e1fd76a..0000000
--- a/render/fx_renderer/gles2/shaders/stencil_mask.frag
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifdef GL_FRAGMENT_PRECISION_HIGH
-precision highp float;
-#else
-precision mediump float;
-#endif
-
-varying vec2 v_texcoord;
-
-uniform vec2 half_size;
-uniform vec2 position;
-uniform float radius;
-
-void main() {
-	vec2 q = abs(gl_FragCoord.xy - position - half_size) - half_size + radius;
-	float dist = min(max(q.x,q.y), 0.0) + length(max(q, 0.0)) - radius;
-	float smoothedAlpha = 1.0 - smoothstep(-1.0, 0.5, dist);
-	gl_FragColor = mix(vec4(0.0), vec4(1.0), smoothedAlpha);
-
-	if (gl_FragColor.a < 1.0) {
-		discard;
-	}
-}
diff --git a/render/fx_renderer/gles2/shaders/tex.frag b/render/fx_renderer/gles2/shaders/tex.frag
index d08c95d..1377c00 100644
--- a/render/fx_renderer/gles2/shaders/tex.frag
+++ b/render/fx_renderer/gles2/shaders/tex.frag
@@ -26,7 +26,7 @@ uniform sampler2D tex;
 
 uniform float alpha;
 
-uniform vec2 size;
+uniform vec2 half_size;
 uniform vec2 position;
 uniform float radius;
 uniform bool has_titlebar;
@@ -42,19 +42,21 @@ vec4 sample_texture() {
 #endif
 }
 
+float roundRectSDF() {
+	vec2 q = abs(gl_FragCoord.xy - position - half_size) - half_size + radius;
+	return min(max(q.x, q.y), 0.0) + length(max(q, 0.0)) - radius;
+}
+
 void main() {
 	gl_FragColor = mix(sample_texture(), dim_color, dim) * alpha;
 
-	if (!has_titlebar || gl_FragCoord.y - position.y > radius) {
-		vec2 corner_distance = min(gl_FragCoord.xy - position, size + position - gl_FragCoord.xy);
-		if (max(corner_distance.x, corner_distance.y) < radius) {
-			float d = radius - distance(corner_distance, vec2(radius));
-			float smooth = smoothstep(-1.0, 0.5, d);
-			gl_FragColor = mix(vec4(0), gl_FragColor, smooth);
-		}
-	}
-
 	if (discard_transparent && gl_FragColor.a == 0.0) {
 		discard;
+		return;
+	}
+
+	if (!has_titlebar || gl_FragCoord.y - position.y > radius) {
+		float alpha = smoothstep(-1.0, 1.0, roundRectSDF());
+		gl_FragColor = mix(gl_FragColor, vec4(0.0), alpha);
 	}
 }