Multimodal Applications › Video Frame Sampling and Temporal Analysis

def simulate_video_scene(n_frames=60, scene_changes_at=None):
    """
    Simulate a video as a sequence of synthetic scene frames.
    Each frame is represented as a feature vector (simulating CNN output).
    Scene changes introduce discontinuous shifts in the feature distribution.
    """
    rng = np.random.default_rng(42)
    if scene_changes_at is None:
        scene_changes_at = [20, 40]

    frames = []
    current_scene = 0
    scene_centers = [
        np.array([0.8, 0.2, 0.5, 0.9, 0.1]),  # Scene 0: outdoor activity
        np.array([0.3, 0.7, 0.8, 0.2, 0.6]),  # Scene 1: indoor conversation
        np.array([0.6, 0.4, 0.2, 0.7, 0.9]),  # Scene 2: sports action
    ]

    for t in range(n_frames):
        # Check for scene transitions
        if (
            current_scene < len(scene_changes_at)
            and t >= scene_changes_at[current_scene]
        ):
            current_scene += 1

        # Frame features: scene center + temporal drift + noise
        scene_center = scene_centers[current_scene]
        temporal_drift = 0.05 * np.sin(2 * np.pi * t / 20)
        noise = rng.normal(0, 0.08, size=5)
        frame_features = np.clip(scene_center + temporal_drift + noise, 0, 1)
        frames.append(frame_features)

    return np.array(frames), scene_changes_at


video_frames, scene_changes = simulate_video_scene()
n_frames = len(video_frames)

def uniform_sampling(frames, n_samples):
    """Sample n_samples frames at uniform intervals."""
    indices = np.linspace(0, len(frames) - 1, n_samples, dtype=int)
    return indices, frames[indices]


def adaptive_sampling(frames, n_samples):
    """
    Sample frames at moments of high temporal change.
    Computes L2 distance between consecutive frames and
    selects frames with highest local change scores.
    """
    # Compute frame-to-frame distances
    distances = np.linalg.norm(np.diff(frames, axis=0), axis=1)

    # Pad to match frame count
    distances = np.append(distances, distances[-1])

    # Score each frame by its distance to neighbors
    smoothed = np.convolve(distances, np.array([0.25, 0.5, 0.25]), mode="same")

    # Always include first and last frame
    scores = smoothed.copy()
    scores[0] = float("inf")
    scores[-1] = float("inf")

    # Greedily select highest-scoring frames
    selected = set([0, len(frames) - 1])
    while len(selected) < n_samples:
        remaining = [i for i in range(len(frames)) if i not in selected]
        next_idx = max(remaining, key=lambda i: scores[i])
        selected.add(next_idx)

    indices = sorted(selected)
    return np.array(indices), frames[indices]


n_samples = 10
uniform_idx, uniform_samples = uniform_sampling(video_frames, n_samples)
adaptive_idx, adaptive_samples = adaptive_sampling(video_frames, n_samples)