nodes: mask off-frame samples to transparent in the distort shaders

Translating, rotating or warping content past the frame edge used to
smear the clamped edge row/column across the vacated region. The
transform, position, swirl, ripple and wave shaders now multiply the
sample by an in-bounds mask so off-frame pixels come out transparent
(and composite as black when nothing sits below). Tile deliberately
keeps its wrapping lookup.

crates/oak-node/src/nodes/position.rs

@@ -61,8 +61,12 @@ void main(void) {
     vec2 half_res = resolution_in * 0.5;
     vec2 px = ove_texcoord * resolution_in - half_res;
     vec2 offset = floor(offset_in + 0.5);
-
-    frag_color = texture(tex_in, (px - offset + half_res) / resolution_in);
+    vec2 uv = (px - offset + half_res) / resolution_in;
+    // Whole-pixel translation past the frame edge leaves transparent
+    // pixels, not the clamped edge column.
+    vec4 col = texture(tex_in, uv);
+    float inside = step(0.0, uv.x) * step(0.0, uv.y) * step(uv.x, 1.0) * step(uv.y, 1.0);
+    frag_color = col * inside;
 }
 "#;
 
@@ -290,7 +294,9 @@ mod tests {
         assert!(code.contains("uniform sampler2D tex_in;"));
         assert!(code.contains("uniform vec2 offset_in;"));
         assert!(code.contains("uniform vec2 resolution_in;"));
-        assert!(code.contains("frag_color = texture(tex_in, (px - offset + half_res) / resolution_in);"));
+        assert!(code.contains("vec2 uv = (px - offset + half_res) / resolution_in;"));
+        // Off-frame samples are masked to transparent, not clamped.
+        assert!(code.contains("frag_color = col * inside;"));
         assert!(!code.contains("switch"));
     }
 

crates/oak-node/src/nodes/rippledistortnode.rs

@@ -93,7 +93,11 @@ void main(void) {
 
   float len = length(adj_texcoord);
   vec2 uv = ove_texcoord + (adj_texcoord/len)*cos((frequency_in)*(len*12.0-evolution_in))*(intensity_in*0.0005);
-  frag_color = texture(tex_in, uv);
+  // Displacement past the frame edge leaves transparent pixels, not the
+  // clamped edge pixels.
+  vec4 col = texture(tex_in, uv);
+  float inside = step(0.0, uv.x) * step(0.0, uv.y) * step(uv.x, 1.0) * step(uv.y, 1.0);
+  frag_color = col * inside;
 }
 "#;
 

crates/oak-node/src/nodes/swirldistortnode.rs

@@ -76,7 +76,12 @@ void main(void) {
     tc = vec2(dot(tc, vec2(c, -s)), dot(tc, vec2(s, c)));
   }
   tc += center;
-  frag_color = texture(tex_in, tc / resolution_in);
+  vec2 uv = tc / resolution_in;
+  // A swirl displacing content past the frame edge leaves transparent
+  // pixels, not the clamped edge pixels.
+  vec4 col = texture(tex_in, uv);
+  float inside = step(0.0, uv.x) * step(0.0, uv.y) * step(uv.x, 1.0) * step(uv.y, 1.0);
+  frag_color = col * inside;
 }
 "#;
 

crates/oak-node/src/nodes/transformdistortnode.rs

@@ -42,7 +42,13 @@ void main(void) {
     vec2 half_res = resolution_in * 0.5;
     vec2 px = ove_texcoord * resolution_in - half_res;
     vec2 src = (transform_in * vec4(px, 0.0, 1.0)).xy + half_res;
-    frag_color = texture(tex_in, src / resolution_in);
+    vec2 uv = src / resolution_in;
+    // A transform that pushes content off-frame leaves the vacated
+    // region TRANSPARENT (the alternative — the sampler's edge clamp —
+    // smears the border pixels across it).
+    vec4 col = texture(tex_in, uv);
+    float inside = step(0.0, uv.x) * step(0.0, uv.y) * step(uv.x, 1.0) * step(uv.y, 1.0);
+    frag_color = col * inside;
 }
 "#;
 

crates/oak-node/src/nodes/wavedistortnode.rs

@@ -69,8 +69,11 @@ void main(void) {
     pos.y -= sin((ove_texcoord.x-(evolution_in*0.01))*frequency_in)*intensity_in*0.01;
   }
 
+  // Out-of-frame displacement leaves transparent pixels (never the
+  // clamped edge pixels, and never `discard`, which would leave the
+  // destination texel undefined).
   if (pos.x < 0.0 || pos.x >= 1.0 || pos.y < 0.0 || pos.y >= 1.0) {
-    discard;
+    frag_color = vec4(0.0);
   } else {
     frag_color = texture(tex_in, pos);
   }

crates/oak-render/src/eval.rs

@@ -3649,6 +3649,47 @@ mod tests {
         );
     }
 
+    /// A transform that pushes content past the frame edge leaves the
+    /// vacated region TRANSPARENT (no edge-pixel smearing): translating
+    /// everything +100px in x empties the frame entirely, and a +3px
+    /// translation vacates exactly the left three columns.
+    #[test]
+    fn gpu_transform_off_frame_is_transparent() {
+        if oak_core::backend::GpuContext::shared().is_none() {
+            eprintln!("no adapter; skipping");
+            return;
+        }
+        let white = filled_frame((8, 8), [1.0, 1.0, 1.0, 1.0]);
+        let mut inputs = NodeValueRow::new();
+        inputs.insert("tex_in".into(), texture_value(white));
+        inputs.insert("pos_in".into(), NodeValue::Vec2([100.0, 0.0]));
+        let out = eval_node_row("org.olivevideoeditor.Olive.transform", inputs, None);
+        for y in 0..8usize {
+            for x in 0..8usize {
+                assert_eq!(
+                    pixel_at(&out, x, y),
+                    [0.0, 0.0, 0.0, 0.0],
+                    "off-frame content must be transparent, got {:?} at ({x},{y})",
+                    pixel_at(&out, x, y)
+                );
+            }
+        }
+
+        let white = filled_frame((8, 8), [1.0, 1.0, 1.0, 1.0]);
+        let mut inputs = NodeValueRow::new();
+        inputs.insert("tex_in".into(), texture_value(white));
+        inputs.insert("pos_in".into(), NodeValue::Vec2([3.0, 0.0]));
+        let out = eval_node_row("org.olivevideoeditor.Olive.transform", inputs, None);
+        for x in 0..3usize {
+            assert_eq!(
+                pixel_at(&out, x, 4),
+                [0.0, 0.0, 0.0, 0.0],
+                "the vacated left columns are transparent"
+            );
+        }
+        assert_eq!(pixel_at(&out, 7, 4), [1.0, 1.0, 1.0, 1.0], "the rightmost column keeps content");
+    }
+
     /// Shape generator over the real GPU path: a centered 8x8 rectangle
     /// on a 16x16 frame fills exactly the middle block (pixel centers
     /// with texcoord in [0.25, 0.75)), everything outside stays

crates/oak-render/tests/ofxmisc_gen.rs

@@ -95,8 +95,18 @@ fn position_shifts_a_white_pixel_by_whole_pixels() {
     row.insert("offset_in".to_string(), NodeValue::Vec2([3.0, 2.0]));
     let frame = eval_node_row(POSITION, row, None);
     assert_pixel(pixel_at(&frame, 5, 5), WHITE, "position moved pixel");
-    assert_pixel(pixel_at(&frame, 2, 3), BLACK, "position vacated pixel");
-    assert_pixel(pixel_at(&frame, 0, 0), BLACK, "position untouched corner");
+    // Off-frame reads are transparent (never the clamped edge pixels):
+    // the composite below shows through instead.
+    assert_pixel(
+        pixel_at(&frame, 2, 3),
+        [0.0, 0.0, 0.0, 0.0],
+        "position vacated pixel",
+    );
+    assert_pixel(
+        pixel_at(&frame, 0, 0),
+        [0.0, 0.0, 0.0, 0.0],
+        "position untouched corner",
+    );
 }
 
 /// Mirror with `horizontal_in` on flips a one-sided white block about the
@@ -211,3 +221,26 @@ fn ramp_from_black_to_white_is_half_at_the_midpoint() {
     );
     assert_pixel(pixel_at(&frame, 7, 3), WHITE, "ramp at point1");
 }
+
+/// Whole-pixel translation past the frame edge leaves transparent
+/// pixels (never the clamped edge column): +100px empties the frame.
+#[test]
+fn position_off_frame_is_transparent() {
+    if !gpu() {
+        eprintln!("no adapter; skipping");
+        return;
+    }
+    let mut row = NodeValueRow::new();
+    row.insert("tex_in".to_string(), texture_value(filled_frame((8, 8), WHITE)));
+    row.insert("offset_in".to_string(), NodeValue::Vec2([100.0, 0.0]));
+    let frame = eval_node_row(POSITION, row, None);
+    for y in 0..8usize {
+        for x in 0..8usize {
+            assert_pixel(
+                pixel_at(&frame, x, y),
+                [0.0, 0.0, 0.0, 0.0],
+                "off-frame content must be transparent",
+            );
+        }
+    }
+}