renderbox_sdk/ops/

timeline.rs

use std::rc::Rc;

use crate::ops::{add_video_filter, params};
use crate::ops::combine::{xfade, XfadeTransition};
use crate::ops::video::{scale, pad, format};
use crate::sorts::Video;
use crate::graph::Stream;

#[derive(Debug, Clone, Copy)]
pub struct Transition {
    pub kind: XfadeTransition,
    pub duration: f64,
}

impl Transition {
    pub fn dissolve(duration: f64) -> Self {
        Self { kind: XfadeTransition::Dissolve, duration }
    }

    pub fn fade(duration: f64) -> Self {
        Self { kind: XfadeTransition::Fade, duration }
    }

    pub fn wipeleft(duration: f64) -> Self {
        Self { kind: XfadeTransition::Wipeleft, duration }
    }

    pub fn wiperight(duration: f64) -> Self {
        Self { kind: XfadeTransition::Wiperight, duration }
    }

    pub fn radial(duration: f64) -> Self {
        Self { kind: XfadeTransition::Radial, duration }
    }

    pub fn pixelize(duration: f64) -> Self {
        Self { kind: XfadeTransition::Pixelize, duration }
    }

    pub fn circleopen(duration: f64) -> Self {
        Self { kind: XfadeTransition::Circleopen, duration }
    }

    pub fn circleclose(duration: f64) -> Self {
        Self { kind: XfadeTransition::Circleclose, duration }
    }
}

#[derive(Debug, Clone, Copy)]
pub enum KenBurnsPreset {
    GentleZoomIn,
    GentleZoomOut,
    GentlePan,
    ZoomIn,
    ZoomOut,
    PanRight,
    PanLeft,
    PunchIn,
    PullBack,
    Spiral,
    Diagonal,
    Parallax,
}

const CENTER_X: &str = "iw/2-(iw/zoom/2)";
const CENTER_Y: &str = "ih/2-(ih/zoom/2)";
const EDGE_X: &str = "iw-(iw/zoom)";
const EDGE_Y: &str = "ih-(ih/zoom)";
const DRIFT_X: &str = "iw/6";
const DRIFT_Y: &str = "ih/6";

fn lerp_expr(frames: u32, from: &str, to: &str) -> String {
    format!("{from}+({to}-({from}))*on/{frames}")
}

fn ease_in_out_expr(frames: u32, from: &str, to: &str) -> String {
    format!(
        "{from}+({to}-({from}))*(3*pow(on/{frames},2)-2*pow(on/{frames},3))"
    )
}

struct KenBurnsExprs {
    zoom: String,
    x: String,
    y: String,
}

fn preset_exprs(preset: KenBurnsPreset, frames: u32) -> KenBurnsExprs {
    match preset {
        KenBurnsPreset::GentleZoomIn => KenBurnsExprs {
            zoom: lerp_expr(frames, "1", "1.15"),
            x: CENTER_X.into(),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::GentleZoomOut => KenBurnsExprs {
            zoom: lerp_expr(frames, "1.15", "1"),
            x: CENTER_X.into(),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::GentlePan => KenBurnsExprs {
            zoom: "1.05".into(),
            x: ease_in_out_expr(frames, "0", CENTER_X),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::ZoomIn => KenBurnsExprs {
            zoom: ease_in_out_expr(frames, "1", "1.5"),
            x: CENTER_X.into(),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::ZoomOut => KenBurnsExprs {
            zoom: ease_in_out_expr(frames, "1.5", "1"),
            x: CENTER_X.into(),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::PanRight => KenBurnsExprs {
            zoom: "1.1".into(),
            x: ease_in_out_expr(frames, "0", EDGE_X),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::PanLeft => KenBurnsExprs {
            zoom: "1.1".into(),
            x: ease_in_out_expr(frames, EDGE_X, "0"),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::PunchIn => KenBurnsExprs {
            zoom: ease_in_out_expr(frames, "1", "2"),
            x: CENTER_X.into(),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::PullBack => KenBurnsExprs {
            zoom: ease_in_out_expr(frames, "2", "1"),
            x: CENTER_X.into(),
            y: CENTER_Y.into(),
        },
        KenBurnsPreset::Spiral => KenBurnsExprs {
            zoom: ease_in_out_expr(frames, "1", "1.4"),
            x: format!(
                "{CENTER_X}+{DRIFT_X}*sin(2*PI*on/{frames})"
            ),
            y: format!(
                "{CENTER_Y}+{DRIFT_Y}*cos(2*PI*on/{frames})"
            ),
        },
        KenBurnsPreset::Diagonal => KenBurnsExprs {
            zoom: ease_in_out_expr(frames, "1", "1.3"),
            x: ease_in_out_expr(frames, "0", EDGE_X),
            y: ease_in_out_expr(frames, "0", EDGE_Y),
        },
        KenBurnsPreset::Parallax => KenBurnsExprs {
            zoom: "1.2".into(),
            x: ease_in_out_expr(frames, DRIFT_X, &format!("{EDGE_X}-{DRIFT_X}")),
            y: CENTER_Y.into(),
        },
    }
}

fn apply_ken_burns(
    input: Stream<Video>,
    preset: KenBurnsPreset,
    duration: f64,
    w: u32,
    h: u32,
    fps: u32,
) -> Stream<Video> {
    let frames = (duration * fps as f64) as u32;
    let exprs = preset_exprs(preset, frames);
    let s = format!("{w}x{h}");

    let doubled_w = w * 2;
    let doubled_h = h * 2;

    let stream = input
        .pipe(scale(doubled_w, doubled_h))
        .pipe(pad(doubled_w, doubled_h));

    let stream = add_video_filter(
        stream,
        "zoompan",
        params! {
            "zoom" => exprs.zoom,
            "x" => exprs.x,
            "y" => exprs.y,
            "d" => frames,
            "s" => s,
            "fps" => fps
        },
    );

    stream.pipe(format("yuv420p"))
}

pub fn ken_burns(
    preset: KenBurnsPreset,
    duration: f64,
    w: u32,
    h: u32,
) -> impl Fn(Stream<Video>) -> Stream<Video> {
    move |input| apply_ken_burns(input, preset, duration, w, h, 25)
}

pub fn ken_burns_with_fps(
    preset: KenBurnsPreset,
    duration: f64,
    w: u32,
    h: u32,
    fps: u32,
) -> impl Fn(Stream<Video>) -> Stream<Video> {
    move |input| apply_ken_burns(input, preset, duration, w, h, fps)
}

pub fn timeline(
    clips: Vec<Stream<Video>>,
    clip_durations: &[f64],
    transition: &Transition,
) -> Stream<Video> {
    assert!(
        clips.len() >= 2,
        "timeline requires at least 2 clips"
    );
    assert_eq!(
        clips.len(),
        clip_durations.len(),
        "clips and clip_durations must have the same length"
    );
    let graph = clips[0].graph.clone();
    for s in &clips[1..] {
        assert!(
            Rc::ptr_eq(&graph, &s.graph),
            "all clips must belong to the same graph"
        );
    }

    let mut acc = clips.into_iter();
    let mut result = acc.next().unwrap();
    let mut elapsed_content = clip_durations[0];
    let mut elapsed_transitions = 0.0;

    for (i, clip) in acc.enumerate() {
        let offset = elapsed_content - elapsed_transitions;
        result = xfade(result, clip, transition.kind, transition.duration, offset);
        elapsed_content += clip_durations[i + 1];
        elapsed_transitions += transition.duration;
    }

    result
}

pub fn slideshow(
    images: Vec<Stream<Video>>,
    segment_duration: f64,
    transition: &Transition,
    preset: KenBurnsPreset,
    w: u32,
    h: u32,
) -> Stream<Video> {
    assert!(
        images.len() >= 2,
        "slideshow requires at least 2 images"
    );
    let graph = images[0].graph.clone();
    for s in &images[1..] {
        assert!(
            Rc::ptr_eq(&graph, &s.graph),
            "all images must belong to the same graph"
        );
    }

    let clips: Vec<Stream<Video>> = images
        .into_iter()
        .map(|img| img.pipe(ken_burns(preset, segment_duration, w, h)))
        .collect();

    let durations: Vec<f64> = vec![segment_duration; clips.len()];
    timeline(clips, &durations, transition)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::graph::input;
    use renderbox_dsl::{OpNode, ParamValue};

    #[test]
    fn transition_constructors() {
        let t = Transition::dissolve(1.5);
        assert!(matches!(t.kind, XfadeTransition::Dissolve));
        assert_eq!(t.duration, 1.5);

        let t = Transition::fade(0.5);
        assert!(matches!(t.kind, XfadeTransition::Fade));
        assert_eq!(t.duration, 0.5);

        let t = Transition::radial(2.0);
        assert!(matches!(t.kind, XfadeTransition::Radial));
        assert_eq!(t.duration, 2.0);
    }

    #[test]
    fn ken_burns_preset_expressions() {
        let exprs = preset_exprs(KenBurnsPreset::GentleZoomIn, 125);
        assert_eq!(exprs.zoom, "1+(1.15-(1))*on/125");
        assert_eq!(exprs.x, CENTER_X);
        assert_eq!(exprs.y, CENTER_Y);

        let exprs = preset_exprs(KenBurnsPreset::ZoomIn, 250);
        assert_eq!(
            exprs.zoom,
            "1+(1.5-(1))*(3*pow(on/250,2)-2*pow(on/250,3))"
        );

        let exprs = preset_exprs(KenBurnsPreset::PanRight, 100);
        assert_eq!(exprs.zoom, "1.1");
        assert_eq!(
            exprs.x,
            format!("0+({EDGE_X}-(0))*(3*pow(on/100,2)-2*pow(on/100,3))")
        );

        let exprs = preset_exprs(KenBurnsPreset::Spiral, 200);
        assert!(exprs.x.contains("sin(2*PI*on/200)"));
        assert!(exprs.y.contains("cos(2*PI*on/200)"));
    }

    #[test]
    fn ken_burns_node_chain() {
        let (v, _a) = input("photo.jpg");
        let result = v.pipe(ken_burns(KenBurnsPreset::GentleZoomIn, 5.0, 1920, 1080));
        let graph = result.graph.borrow();

        // Walk backwards: format <- zoompan <- pad <- scale <- input
        match graph.arena.get(result.node_id) {
            OpNode::Filter { name, .. } => assert_eq!(name, "format"),
            other => panic!("expected format Filter, got {:?}", other),
        }

        // Find zoompan by checking node count: input(0) + scale(1) + pad(2) + zoompan(3) + format(4)
        assert_eq!(graph.arena.len(), 5);
    }

    #[test]
    fn ken_burns_zoompan_params() {
        let (v, _a) = input("photo.jpg");
        let result = v.pipe(ken_burns(KenBurnsPreset::GentleZoomIn, 5.0, 1920, 1080));
        let graph = result.graph.borrow();

        // format node's input is the zoompan node
        let format_input = match graph.arena.get(result.node_id) {
            OpNode::Filter { input, .. } => *input,
            other => panic!("expected format Filter, got {:?}", other),
        };
        match graph.arena.get(format_input) {
            OpNode::Filter { name, params, .. } => {
                assert_eq!(name, "zoompan");
                assert_eq!(params["s"], ParamValue::String("1920x1080".into()));
                assert_eq!(params["d"], ParamValue::Number(125.0));
                assert_eq!(params["fps"], ParamValue::Number(25.0));
            }
            other => panic!("expected zoompan Filter, got {:?}", other),
        }
    }

    #[test]
    fn timeline_two_clips() {
        let (v, _a) = input("test.mp4");
        let v1 = v.clone();
        let v2 = v.clone();
        let t = Transition::dissolve(1.0);
        let result = timeline(vec![v1, v2], &[5.0, 5.0], &t);

        let graph = result.graph.borrow();
        match graph.arena.get(result.node_id) {
            OpNode::Combine2 { op, params, .. } => {
                assert_eq!(op, "xfade");
                assert_eq!(params["transition"], ParamValue::String("dissolve".into()));
                assert_eq!(params["duration"], ParamValue::Number(1.0));
                assert_eq!(params["offset"], ParamValue::Number(5.0));
            }
            other => panic!("expected Combine2 xfade, got {:?}", other),
        }
    }

    #[test]
    fn timeline_three_clips() {
        let (v, _a) = input("test.mp4");
        let v1 = v.clone();
        let v2 = v.clone();
        let v3 = v.clone();
        let t = Transition::dissolve(1.0);
        let result = timeline(vec![v1, v2, v3], &[5.0, 5.0, 5.0], &t);

        let graph = result.graph.borrow();
        // Final node is the second xfade
        match graph.arena.get(result.node_id) {
            OpNode::Combine2 { op, params, .. } => {
                assert_eq!(op, "xfade");
                // offset for clip 3: (5.0 + 5.0) - 1.0 = 9.0
                assert_eq!(params["offset"], ParamValue::Number(9.0));
            }
            other => panic!("expected Combine2 xfade, got {:?}", other),
        }

        // Should have: input(0), xfade1(1), xfade2(2) = 3 nodes
        assert_eq!(graph.arena.len(), 3);
    }

    #[test]
    fn slideshow_produces_ken_burns_and_xfade() {
        let (v, _a) = input("test.mp4");
        let images: Vec<Stream<Video>> = (0..3).map(|_| v.clone()).collect();
        let t = Transition::dissolve(1.0);
        let result = slideshow(images, 5.0, &t, KenBurnsPreset::GentleZoomIn, 1920, 1080);

        let graph = result.graph.borrow();
        // Final node should be an xfade
        match graph.arena.get(result.node_id) {
            OpNode::Combine2 { op, .. } => assert_eq!(op, "xfade"),
            other => panic!("expected Combine2 xfade, got {:?}", other),
        }

        // 1 input + 3*(scale+pad+zoompan+format) + 2 xfades = 1 + 12 + 2 = 15
        assert_eq!(graph.arena.len(), 15);
    }

    #[test]
    #[should_panic(expected = "timeline requires at least 2 clips")]
    fn timeline_requires_at_least_2() {
        let (v, _a) = input("test.mp4");
        let t = Transition::dissolve(1.0);
        timeline(vec![v], &[5.0], &t);
    }

    #[test]
    #[should_panic(expected = "slideshow requires at least 2 images")]
    fn slideshow_requires_at_least_2() {
        let (v, _a) = input("test.mp4");
        let t = Transition::dissolve(1.0);
        slideshow(vec![v], 5.0, &t, KenBurnsPreset::GentleZoomIn, 1920, 1080);
    }

    #[test]
    fn lerp_expr_format() {
        assert_eq!(lerp_expr(100, "1", "2"), "1+(2-(1))*on/100");
    }

    #[test]
    fn ease_in_out_expr_format() {
        assert_eq!(
            ease_in_out_expr(100, "1", "2"),
            "1+(2-(1))*(3*pow(on/100,2)-2*pow(on/100,3))"
        );
    }
}