feat: add thumbnail extraction and timeline display

Wire ffmpeg frame extraction into post-download processing and render
adaptive thumbnail strips on the timeline via the asset protocol.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/lib/bindings/mediaAnalysis.ts

@@ -20,3 +20,13 @@ export async function extractWaveform(
 ): Promise<number[]> {
   return invoke<number[]>('extract_waveform', { filePath, sampleCount });
 }
+
+export async function extractThumbnails(
+  filePath: string,
+  duration: number
+): Promise<ThumbnailSpritesheet[]> {
+  return invoke<ThumbnailSpritesheet[]>('extract_thumbnails', {
+    filePath,
+    duration,
+  });
+}

src/lib/stores/videoSession.svelte.ts

@@ -1,7 +1,11 @@
 import type { VideoMetadata } from '$lib/bindings/video';
 import { startDownload } from '$lib/bindings/video';
 import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis';
-import { extractKeyframes, extractWaveform } from '$lib/bindings/mediaAnalysis';
+import {
+  extractKeyframes,
+  extractThumbnails,
+  extractWaveform,
+} from '$lib/bindings/mediaAnalysis';
 
 export type SessionStatus = 'idle' | 'resolving' | 'ready' | 'error';
 export type DownloadStatus = 'idle' | 'downloading' | 'complete' | 'failed';
@@ -79,9 +83,11 @@ export function reset() {
 }
 
 async function triggerPostDownloadProcessing(filePath: string) {
-  const [keyframes, waveform] = await Promise.allSettled([
+  const duration = session.duration;
+  const [keyframes, waveform, thumbnails] = await Promise.allSettled([
     extractKeyframes(filePath),
     extractWaveform(filePath, 8000),
+    extractThumbnails(filePath, duration),
   ]);
 
   if (keyframes.status === 'fulfilled') {
@@ -95,6 +101,12 @@ async function triggerPostDownloadProcessing(filePath: string) {
   } else {
     console.error('Waveform extraction failed:', waveform.reason);
   }
+
+  if (thumbnails.status === 'fulfilled') {
+    session.thumbnailSpritesheets = thumbnails.value;
+  } else {
+    console.error('Thumbnail extraction failed:', thumbnails.reason);
+  }
 }
 
 export async function beginDownload() {
@@ -105,6 +117,7 @@ export async function beginDownload() {
   session.localFilePath = null;
   session.keyframePositions = [];
   session.waveformPeaks = [];
+  session.thumbnailSpritesheets = [];
 
   try {
     await startDownload(

src/lib/timeline/renderer.ts

@@ -1,6 +1,7 @@
 import { drawClips } from './clipRenderer';
 import type { Clip } from '$lib/stores/clips.svelte';
 import { drawWaveform } from './waveformRenderer';
+import { drawThumbnails } from './thumbnailRenderer';
 import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis';
 
 export interface TimelineState {
@@ -36,7 +37,7 @@ export function drawTimeline(
   selectedClipId: string | null = null,
   pendingInPoint: number | null = null,
   waveformPeaks: number[] = [],
-  _thumbnailSpritesheets: ThumbnailSpritesheet[] = []
+  thumbnailSpritesheets: ThumbnailSpritesheet[] = []
 ): void {
   const { width, height } = state;
   ctx.clearRect(0, 0, width, height);
@@ -46,7 +47,11 @@ export function drawTimeline(
 
   drawTimeTicks(ctx, state, duration);
 
-  drawPlaceholderLane(ctx, state, 0, THUMB_LANE_HEIGHT, 'Thumbnails');
+  if (thumbnailSpritesheets.length > 0) {
+    drawThumbnails(ctx, state, thumbnailSpritesheets, duration, 0, THUMB_LANE_HEIGHT);
+  } else {
+    drawPlaceholderLane(ctx, state, 0, THUMB_LANE_HEIGHT, 'Thumbnails');
+  }
 
   if (waveformPeaks.length > 0) {
     drawWaveform(ctx, state, waveformPeaks, duration, THUMB_LANE_HEIGHT, WAVEFORM_LANE_HEIGHT);

src/lib/timeline/thumbnailRenderer.ts

@@ -0,0 +1,84 @@
+import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis';
+import { convertFileSrc } from '@tauri-apps/api/core';
+import { timeToX, type TimelineState } from './renderer';
+
+const imageCache = new Map<string, HTMLImageElement>();
+
+const TARGET_SPACING_PX = 100;
+
+function getImagePath(dir: string, frameIndex: number): string {
+  const paddedIndex = String(frameIndex + 1).padStart(6, '0');
+  return `${dir}/frame_${paddedIndex}.jpg`;
+}
+
+function loadImage(path: string): void {
+  if (imageCache.has(path)) {
+    return;
+  }
+
+  const img = new Image();
+  img.src = convertFileSrc(path);
+  img.onload = () => imageCache.set(path, img);
+  imageCache.set(path, img);
+}
+
+function findSheetForFrame(
+  spritesheets: ThumbnailSpritesheet[],
+  frameIndex: number
+): ThumbnailSpritesheet | null {
+  for (const sheet of spritesheets) {
+    if (frameIndex >= sheet.startIndex && frameIndex < sheet.startIndex + sheet.count) {
+      return sheet;
+    }
+  }
+  return null;
+}
+
+export function drawThumbnails(
+  ctx: CanvasRenderingContext2D,
+  state: TimelineState,
+  spritesheets: ThumbnailSpritesheet[],
+  duration: number,
+  y: number,
+  height: number
+): void {
+  if (spritesheets.length === 0) return;
+
+  const sheet = spritesheets[0];
+  const interval = sheet.intervalSeconds;
+  const thumbWidth = Math.max(height * (sheet.thumbWidth / sheet.thumbHeight), 20);
+
+  const visibleRange = state.visibleEnd - state.visibleStart;
+  const framesInView = visibleRange / interval;
+  const maxThumbs = Math.max(1, Math.floor(state.width / TARGET_SPACING_PX));
+  const step = Math.max(1, Math.ceil(framesInView / maxThumbs));
+
+  const startFrame = Math.floor(state.visibleStart / interval);
+  const endFrame = Math.ceil(state.visibleEnd / interval);
+
+  for (let fi = startFrame; fi <= endFrame; fi += step) {
+    const time = fi * interval;
+    if (time < 0 || time > duration) continue;
+
+    const x = timeToX(time, state);
+    if (x < -thumbWidth || x > state.width + thumbWidth) continue;
+
+    const foundSheet = findSheetForFrame(spritesheets, fi);
+    if (!foundSheet) continue;
+
+    const imgPath = getImagePath(foundSheet.filePath, fi);
+    const img = imageCache.get(imgPath);
+    const drawX = x - thumbWidth / 2;
+
+    if (img && img.complete && img.naturalWidth > 0) {
+      ctx.drawImage(img, drawX, y, thumbWidth, height);
+    } else {
+      ctx.fillStyle = '#353550';
+      ctx.fillRect(drawX, y, thumbWidth, height);
+      loadImage(imgPath);
+    }
+
+    ctx.strokeStyle = '#45475a';
+    ctx.strokeRect(drawX, y, thumbWidth, height);
+  }
+}