feat: add waveform generation and timeline display

Co-authored-by: Cursor <cursoragent@cursor.com>

src-tauri/src/commands/media_analysis.rs

@@ -1,4 +1,4 @@
-use crate::services::keyframe_index;
+use crate::services::{keyframe_index, waveform_generator};
 
 #[tauri::command]
 pub async fn extract_keyframes(file_path: String) -> Result<Vec<f64>, String> {
@@ -6,3 +6,15 @@ pub async fn extract_keyframes(file_path: String) -> Result<Vec<f64>, String> {
         .await
         .map_err(|e| format!("Task failed: {e}"))?
 }
+
+#[tauri::command]
+pub async fn extract_waveform(
+    file_path: String,
+    sample_count: usize,
+) -> Result<Vec<f64>, String> {
+    tokio::task::spawn_blocking(move || {
+        waveform_generator::extract_waveform(&file_path, sample_count)
+    })
+    .await
+    .map_err(|e| format!("Task failed: {e}"))?
+}

src-tauri/src/lib.rs

@@ -17,6 +17,7 @@ pub fn run() {
             video::resolve_url,
             video::start_download,
             media_analysis::extract_keyframes,
+            media_analysis::extract_waveform,
         ])
         .run(tauri::generate_context!())
         .expect("error while running tauri application");

src-tauri/src/services/waveform_generator.rs

@@ -1 +1,90 @@
-// Implemented in Task 10
+use std::process::Command;
+
+pub fn extract_waveform(file_path: &str, sample_count: usize) -> Result<Vec<f64>, String> {
+    let output = Command::new("ffmpeg")
+        .args([
+            "-i",
+            file_path,
+            "-ac",
+            "1",
+            "-filter:a",
+            &format!("aresample={sample_count}"),
+            "-f",
+            "f32le",
+            "-vn",
+            "-",
+        ])
+        .output()
+        .map_err(|e| format!("Failed to run ffmpeg: {e}"))?;
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        return Err(format!("ffmpeg waveform extraction failed: {stderr}"));
+    }
+
+    let samples = parse_f32_samples(&output.stdout);
+    let peaks = compute_peaks(&samples, sample_count);
+    Ok(peaks)
+}
+
+fn parse_f32_samples(data: &[u8]) -> Vec<f32> {
+    data.chunks_exact(4)
+        .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]))
+        .collect()
+}
+
+fn compute_peaks(samples: &[f32], target_count: usize) -> Vec<f64> {
+    if samples.is_empty() || target_count == 0 {
+        return vec![0.0; target_count];
+    }
+
+    let chunk_size = (samples.len() as f64 / target_count as f64).ceil() as usize;
+    let chunk_size = chunk_size.max(1);
+
+    let mut peaks: Vec<f64> = samples
+        .chunks(chunk_size)
+        .map(|chunk| chunk.iter().map(|s| s.abs() as f64).fold(0.0_f64, f64::max))
+        .collect();
+
+    let max_peak = peaks.iter().cloned().fold(0.0_f64, f64::max);
+    if max_peak > 0.0 {
+        for peak in &mut peaks {
+            *peak /= max_peak;
+        }
+    }
+
+    peaks.resize(target_count, 0.0);
+    peaks
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_compute_peaks_basic() {
+        let samples = vec![0.5, -0.8, 0.3, -0.1, 0.9, -0.2];
+        let peaks = compute_peaks(&samples, 3);
+        assert_eq!(peaks.len(), 3);
+        assert!(peaks[0] > 0.0);
+        assert!(peaks[1] > 0.0);
+        assert!(peaks[2] > 0.0);
+        assert!((peaks[2] - 1.0).abs() < 0.001);
+    }
+
+    #[test]
+    fn test_compute_peaks_empty() {
+        let peaks = compute_peaks(&[], 5);
+        assert_eq!(peaks.len(), 5);
+        assert!(peaks.iter().all(|p| *p == 0.0));
+    }
+
+    #[test]
+    fn test_parse_f32_samples() {
+        let val: f32 = 0.5;
+        let bytes = val.to_le_bytes();
+        let samples = parse_f32_samples(&bytes);
+        assert_eq!(samples.len(), 1);
+        assert!((samples[0] - 0.5).abs() < 0.001);
+    }
+}

src/lib/bindings/mediaAnalysis.ts

@@ -13,3 +13,10 @@ export interface ThumbnailSpritesheet {
 export async function extractKeyframes(filePath: string): Promise<number[]> {
   return invoke<number[]>('extract_keyframes', { filePath });
 }
+
+export async function extractWaveform(
+  filePath: string,
+  sampleCount: number
+): Promise<number[]> {
+  return invoke<number[]>('extract_waveform', { filePath, sampleCount });
+}

src/lib/components/Timeline.svelte

@@ -69,7 +69,9 @@
         session.duration,
         clipStore.clips,
         clipStore.selectedClipId,
-        clipStore.pendingInPoint
+        clipStore.pendingInPoint,
+        session.waveformPeaks,
+        session.thumbnailSpritesheets
       );
       ctx.restore();
       animFrameId = requestAnimationFrame(draw);

src/lib/stores/videoSession.svelte.ts

@@ -1,7 +1,7 @@
 import type { VideoMetadata } from '$lib/bindings/video';
 import { startDownload } from '$lib/bindings/video';
 import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis';
-import { extractKeyframes } from '$lib/bindings/mediaAnalysis';
+import { extractKeyframes, extractWaveform } from '$lib/bindings/mediaAnalysis';
 
 export type SessionStatus = 'idle' | 'resolving' | 'ready' | 'error';
 export type DownloadStatus = 'idle' | 'downloading' | 'complete' | 'failed';
@@ -79,12 +79,22 @@ export function reset() {
 }
 
 async function triggerPostDownloadProcessing(filePath: string) {
-  try {
+  const [keyframes, waveform] = await Promise.allSettled([
-    session.keyframePositions = await extractKeyframes(filePath);
+    extractKeyframes(filePath),
-  } catch (e) {
+    extractWaveform(filePath, 8000),
-    console.error('Keyframe extraction failed:', e);
+  ]);
+
+  if (keyframes.status === 'fulfilled') {
+    session.keyframePositions = keyframes.value;
+  } else {
+    console.error('Keyframe extraction failed:', keyframes.reason);
+  }
+
+  if (waveform.status === 'fulfilled') {
+    session.waveformPeaks = waveform.value;
+  } else {
+    console.error('Waveform extraction failed:', waveform.reason);
   }
-  // Waveform and thumbnails will be added in Tasks 10 and 11
 }
 
 export async function beginDownload() {

src/lib/timeline/renderer.ts

@@ -1,5 +1,7 @@
 import { drawClips } from './clipRenderer';
 import type { Clip } from '$lib/stores/clips.svelte';
+import { drawWaveform } from './waveformRenderer';
+import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis';
 
 export interface TimelineState {
   visibleStart: number;
@@ -32,7 +34,9 @@ export function drawTimeline(
   duration: number,
   clips: Clip[] = [],
   selectedClipId: string | null = null,
-  pendingInPoint: number | null = null
+  pendingInPoint: number | null = null,
+  waveformPeaks: number[] = [],
+  _thumbnailSpritesheets: ThumbnailSpritesheet[] = []
 ): void {
   const { width, height } = state;
   ctx.clearRect(0, 0, width, height);
@@ -43,7 +47,12 @@ export function drawTimeline(
   drawTimeTicks(ctx, state, duration);
 
   drawPlaceholderLane(ctx, state, 0, THUMB_LANE_HEIGHT, 'Thumbnails');
-  drawPlaceholderLane(ctx, state, THUMB_LANE_HEIGHT, WAVEFORM_LANE_HEIGHT, 'Waveform');
+
+  if (waveformPeaks.length > 0) {
+    drawWaveform(ctx, state, waveformPeaks, duration, THUMB_LANE_HEIGHT, WAVEFORM_LANE_HEIGHT);
+  } else {
+    drawPlaceholderLane(ctx, state, THUMB_LANE_HEIGHT, WAVEFORM_LANE_HEIGHT, 'Waveform');
+  }
 
   drawClips(ctx, state, clips, selectedClipId, pendingInPoint);
 

src/lib/timeline/waveformRenderer.ts

@@ -0,0 +1,44 @@
+import type { TimelineState } from './renderer';
+
+export function drawWaveform(
+  ctx: CanvasRenderingContext2D,
+  state: TimelineState,
+  peaks: number[],
+  duration: number,
+  y: number,
+  height: number
+): void {
+  if (peaks.length === 0 || duration <= 0) return;
+
+  const { width, visibleStart, visibleEnd } = state;
+  const visibleRange = visibleEnd - visibleStart;
+
+  ctx.fillStyle = '#89b4fa44';
+  ctx.strokeStyle = '#89b4fa88';
+  ctx.lineWidth = 1;
+
+  ctx.beginPath();
+  ctx.moveTo(0, y + height);
+
+  for (let px = 0; px < width; px++) {
+    const startSample = Math.floor(
+      ((visibleStart + ((px - 0.5) / width) * visibleRange) / duration) * peaks.length
+    );
+    const endSample = Math.floor(
+      ((visibleStart + ((px + 0.5) / width) * visibleRange) / duration) * peaks.length
+    );
+
+    let maxPeak = 0;
+    for (let i = Math.max(0, startSample); i <= Math.min(endSample, peaks.length - 1); i++) {
+      maxPeak = Math.max(maxPeak, peaks[i]);
+    }
+
+    const barHeight = maxPeak * height;
+    ctx.lineTo(px, y + height - barHeight);
+  }
+
+  ctx.lineTo(width, y + height);
+  ctx.closePath();
+  ctx.fill();
+  ctx.stroke();
+}