feat: add thumbnail extraction and timeline display

Wire ffmpeg frame extraction into post-download processing and render
adaptive thumbnail strips on the timeline via the asset protocol.

Co-authored-by: Cursor <cursoragent@cursor.com>

src-tauri/Cargo.lock

@@ -1300,6 +1300,12 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "http-range"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573"
+
 [[package]]
 name = "httparse"
 version = "1.10.1"
@@ -3252,6 +3258,7 @@ dependencies = [
  "gtk",
  "heck 0.5.0",
  "http",
+ "http-range",
  "jni",
  "libc",
  "log",

src-tauri/Cargo.toml

@@ -18,7 +18,7 @@ crate-type = ["staticlib", "cdylib", "rlib"]
 tauri-build = { version = "2", features = [] }
 
 [dependencies]
-tauri = { version = "2", features = [] }
+tauri = { version = "2", features = ["protocol-asset"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tauri-plugin-shell = "2.3.6"

src-tauri/src/commands/media_analysis.rs

@@ -1,4 +1,5 @@
-use crate::services::{keyframe_index, waveform_generator};
+use crate::models::ThumbnailSpritesheet;
+use crate::services::{keyframe_index, thumbnail_extractor, waveform_generator};
 
 #[tauri::command]
 pub async fn extract_keyframes(file_path: String) -> Result<Vec<f64>, String> {
@@ -18,3 +19,19 @@ pub async fn extract_waveform(
     .await
     .map_err(|e| format!("Task failed: {e}"))?
 }
+
+#[tauri::command]
+pub async fn extract_thumbnails(
+    file_path: String,
+    duration: f64,
+) -> Result<Vec<ThumbnailSpritesheet>, String> {
+    tokio::task::spawn_blocking(move || {
+        let temp_dir = std::env::temp_dir()
+            .join("video-clipper-thumbs")
+            .join(uuid::Uuid::new_v4().to_string());
+        let temp_dir_str = temp_dir.to_string_lossy().to_string();
+        thumbnail_extractor::extract_thumbnails(&file_path, duration, &temp_dir_str)
+    })
+    .await
+    .map_err(|e| format!("Task failed: {e}"))?
+}

src-tauri/src/lib.rs

@@ -18,6 +18,7 @@ pub fn run() {
             video::start_download,
             media_analysis::extract_keyframes,
             media_analysis::extract_waveform,
+            media_analysis::extract_thumbnails,
         ])
         .run(tauri::generate_context!())
         .expect("error while running tauri application");

src-tauri/src/services/thumbnail_extractor.rs

@@ -1 +1,97 @@
-// Implemented in Task 11
+use crate::models::ThumbnailSpritesheet;
+use std::process::Command;
+
+pub fn compute_interval(duration: f64) -> f64 {
+    if duration < 600.0 {
+        1.0
+    } else if duration < 3600.0 {
+        2.0
+    } else {
+        5.0
+    }
+}
+
+pub fn extract_thumbnails(
+    file_path: &str,
+    duration: f64,
+    output_dir: &str,
+) -> Result<Vec<ThumbnailSpritesheet>, String> {
+    std::fs::create_dir_all(output_dir)
+        .map_err(|e| format!("Failed to create thumbnail dir: {e}"))?;
+
+    let interval = compute_interval(duration);
+    let thumb_width = 160u32;
+    let thumb_height = 90u32;
+    let columns = 10u32;
+    let rows_per_sheet = 10u32;
+    let frames_per_sheet = (columns * rows_per_sheet) as usize;
+
+    let frames_pattern = format!("{output_dir}/frame_%06d.jpg");
+    let status = Command::new("ffmpeg")
+        .args([
+            "-i",
+            file_path,
+            "-vf",
+            &format!("fps=1/{interval},scale={thumb_width}:{thumb_height}"),
+            "-q:v",
+            "5",
+            &frames_pattern,
+        ])
+        .output()
+        .map_err(|e| format!("Failed to extract thumbnails: {e}"))?;
+
+    if !status.status.success() {
+        let stderr = String::from_utf8_lossy(&status.stderr);
+        return Err(format!("Thumbnail extraction failed: {stderr}"));
+    }
+
+    let actual_frames = std::fs::read_dir(output_dir)
+        .map_err(|e| format!("Failed to read thumbnail dir: {e}"))?
+        .filter_map(|e| e.ok())
+        .filter(|e| {
+            e.path()
+                .file_name()
+                .map(|n| n.to_string_lossy().starts_with("frame_"))
+                .unwrap_or(false)
+        })
+        .count();
+
+    let mut sheets = Vec::new();
+    let mut frame_index = 0;
+
+    while frame_index < actual_frames {
+        let count = std::cmp::min(frames_per_sheet, actual_frames - frame_index);
+        sheets.push(ThumbnailSpritesheet {
+            file_path: output_dir.to_string(),
+            start_index: frame_index,
+            count,
+            thumb_width,
+            thumb_height,
+            columns,
+            interval_seconds: interval,
+        });
+        frame_index += count;
+    }
+
+    Ok(sheets)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_compute_interval_short_video() {
+        assert!((compute_interval(120.0) - 1.0).abs() < 0.001);
+    }
+
+    #[test]
+    fn test_compute_interval_medium_video() {
+        assert!((compute_interval(1800.0) - 2.0).abs() < 0.001);
+    }
+
+    #[test]
+    fn test_compute_interval_long_video() {
+        assert!((compute_interval(7200.0) - 5.0).abs() < 0.001);
+    }
+}

src-tauri/tauri.conf.json

@@ -18,7 +18,11 @@
       }
     ],
     "security": {
-      "csp": null
+      "csp": null,
+      "assetProtocol": {
+        "enable": true,
+        "scope": ["$TEMP/**", "$TMP/**", "/tmp/**"]
+      }
     }
   },
   "bundle": {

src/lib/bindings/mediaAnalysis.ts

@@ -20,3 +20,13 @@ export async function extractWaveform(
 ): Promise<number[]> {
   return invoke<number[]>('extract_waveform', { filePath, sampleCount });
 }
+
+export async function extractThumbnails(
+  filePath: string,
+  duration: number
+): Promise<ThumbnailSpritesheet[]> {
+  return invoke<ThumbnailSpritesheet[]>('extract_thumbnails', {
+    filePath,
+    duration,
+  });
+}

src/lib/stores/videoSession.svelte.ts

@@ -1,7 +1,11 @@
 import type { VideoMetadata } from '$lib/bindings/video';
 import { startDownload } from '$lib/bindings/video';
 import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis';
-import { extractKeyframes, extractWaveform } from '$lib/bindings/mediaAnalysis';
+import {
+  extractKeyframes,
+  extractThumbnails,
+  extractWaveform,
+} from '$lib/bindings/mediaAnalysis';
 
 export type SessionStatus = 'idle' | 'resolving' | 'ready' | 'error';
 export type DownloadStatus = 'idle' | 'downloading' | 'complete' | 'failed';
@@ -79,9 +83,11 @@ export function reset() {
 }
 
 async function triggerPostDownloadProcessing(filePath: string) {
-  const [keyframes, waveform] = await Promise.allSettled([
+  const duration = session.duration;
+  const [keyframes, waveform, thumbnails] = await Promise.allSettled([
     extractKeyframes(filePath),
     extractWaveform(filePath, 8000),
+    extractThumbnails(filePath, duration),
   ]);
 
   if (keyframes.status === 'fulfilled') {
@@ -95,6 +101,12 @@ async function triggerPostDownloadProcessing(filePath: string) {
   } else {
     console.error('Waveform extraction failed:', waveform.reason);
   }
+
+  if (thumbnails.status === 'fulfilled') {
+    session.thumbnailSpritesheets = thumbnails.value;
+  } else {
+    console.error('Thumbnail extraction failed:', thumbnails.reason);
+  }
 }
 
 export async function beginDownload() {
@@ -105,6 +117,7 @@ export async function beginDownload() {
   session.localFilePath = null;
   session.keyframePositions = [];
   session.waveformPeaks = [];
+  session.thumbnailSpritesheets = [];
 
   try {
     await startDownload(

src/lib/timeline/renderer.ts

@@ -1,6 +1,7 @@
 import { drawClips } from './clipRenderer';
 import type { Clip } from '$lib/stores/clips.svelte';
 import { drawWaveform } from './waveformRenderer';
+import { drawThumbnails } from './thumbnailRenderer';
 import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis';
 
 export interface TimelineState {
@@ -36,7 +37,7 @@ export function drawTimeline(
   selectedClipId: string | null = null,
   pendingInPoint: number | null = null,
   waveformPeaks: number[] = [],
-  _thumbnailSpritesheets: ThumbnailSpritesheet[] = []
+  thumbnailSpritesheets: ThumbnailSpritesheet[] = []
 ): void {
   const { width, height } = state;
   ctx.clearRect(0, 0, width, height);
@@ -46,7 +47,11 @@ export function drawTimeline(
 
   drawTimeTicks(ctx, state, duration);
 
+  if (thumbnailSpritesheets.length > 0) {
+    drawThumbnails(ctx, state, thumbnailSpritesheets, duration, 0, THUMB_LANE_HEIGHT);
+  } else {
     drawPlaceholderLane(ctx, state, 0, THUMB_LANE_HEIGHT, 'Thumbnails');
+  }
 
   if (waveformPeaks.length > 0) {
     drawWaveform(ctx, state, waveformPeaks, duration, THUMB_LANE_HEIGHT, WAVEFORM_LANE_HEIGHT);

src/lib/timeline/thumbnailRenderer.ts

@@ -0,0 +1,84 @@
+import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis';
+import { convertFileSrc } from '@tauri-apps/api/core';
+import { timeToX, type TimelineState } from './renderer';
+
+const imageCache = new Map<string, HTMLImageElement>();
+
+const TARGET_SPACING_PX = 100;
+
+function getImagePath(dir: string, frameIndex: number): string {
+  const paddedIndex = String(frameIndex + 1).padStart(6, '0');
+  return `${dir}/frame_${paddedIndex}.jpg`;
+}
+
+function loadImage(path: string): void {
+  if (imageCache.has(path)) {
+    return;
+  }
+
+  const img = new Image();
+  img.src = convertFileSrc(path);
+  img.onload = () => imageCache.set(path, img);
+  imageCache.set(path, img);
+}
+
+function findSheetForFrame(
+  spritesheets: ThumbnailSpritesheet[],
+  frameIndex: number
+): ThumbnailSpritesheet | null {
+  for (const sheet of spritesheets) {
+    if (frameIndex >= sheet.startIndex && frameIndex < sheet.startIndex + sheet.count) {
+      return sheet;
+    }
+  }
+  return null;
+}
+
+export function drawThumbnails(
+  ctx: CanvasRenderingContext2D,
+  state: TimelineState,
+  spritesheets: ThumbnailSpritesheet[],
+  duration: number,
+  y: number,
+  height: number
+): void {
+  if (spritesheets.length === 0) return;
+
+  const sheet = spritesheets[0];
+  const interval = sheet.intervalSeconds;
+  const thumbWidth = Math.max(height * (sheet.thumbWidth / sheet.thumbHeight), 20);
+
+  const visibleRange = state.visibleEnd - state.visibleStart;
+  const framesInView = visibleRange / interval;
+  const maxThumbs = Math.max(1, Math.floor(state.width / TARGET_SPACING_PX));
+  const step = Math.max(1, Math.ceil(framesInView / maxThumbs));
+
+  const startFrame = Math.floor(state.visibleStart / interval);
+  const endFrame = Math.ceil(state.visibleEnd / interval);
+
+  for (let fi = startFrame; fi <= endFrame; fi += step) {
+    const time = fi * interval;
+    if (time < 0 || time > duration) continue;
+
+    const x = timeToX(time, state);
+    if (x < -thumbWidth || x > state.width + thumbWidth) continue;
+
+    const foundSheet = findSheetForFrame(spritesheets, fi);
+    if (!foundSheet) continue;
+
+    const imgPath = getImagePath(foundSheet.filePath, fi);
+    const img = imageCache.get(imgPath);
+    const drawX = x - thumbWidth / 2;
+
+    if (img && img.complete && img.naturalWidth > 0) {
+      ctx.drawImage(img, drawX, y, thumbWidth, height);
+    } else {
+      ctx.fillStyle = '#353550';
+      ctx.fillRect(drawX, y, thumbWidth, height);
+      loadImage(imgPath);
+    }
+
+    ctx.strokeStyle = '#45475a';
+    ctx.strokeRect(drawX, y, thumbWidth, height);
+  }
+}