From e05e83f62f6034c46f74170db0011b3033236a4c Mon Sep 17 00:00:00 2001 From: cottongin Date: Mon, 21 Sep 2026 11:12:36 -0400 Subject: [PATCH] feat: add waveform generation and timeline display Co-authored-by: Cursor --- src-tauri/src/commands/media_analysis.rs | 14 ++- src-tauri/src/lib.rs | 1 + src-tauri/src/services/waveform_generator.rs | 91 +++++++++++++++++++- src/lib/bindings/mediaAnalysis.ts | 7 ++ src/lib/components/Timeline.svelte | 4 +- src/lib/stores/videoSession.svelte.ts | 22 +++-- src/lib/timeline/renderer.ts | 13 ++- src/lib/timeline/waveformRenderer.ts | 44 ++++++++++ 8 files changed, 185 insertions(+), 11 deletions(-) create mode 100644 src/lib/timeline/waveformRenderer.ts diff --git a/src-tauri/src/commands/media_analysis.rs b/src-tauri/src/commands/media_analysis.rs index c813569..aa48180 100644 --- a/src-tauri/src/commands/media_analysis.rs +++ b/src-tauri/src/commands/media_analysis.rs @@ -1,4 +1,4 @@ -use crate::services::keyframe_index; +use crate::services::{keyframe_index, waveform_generator}; #[tauri::command] pub async fn extract_keyframes(file_path: String) -> Result, String> { @@ -6,3 +6,15 @@ pub async fn extract_keyframes(file_path: String) -> Result, String> { .await .map_err(|e| format!("Task failed: {e}"))? } + +#[tauri::command] +pub async fn extract_waveform( + file_path: String, + sample_count: usize, +) -> Result, String> { + tokio::task::spawn_blocking(move || { + waveform_generator::extract_waveform(&file_path, sample_count) + }) + .await + .map_err(|e| format!("Task failed: {e}"))? +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 33d0e22..aeecfc8 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -17,6 +17,7 @@ pub fn run() { video::resolve_url, video::start_download, media_analysis::extract_keyframes, + media_analysis::extract_waveform, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/src/services/waveform_generator.rs b/src-tauri/src/services/waveform_generator.rs index d7b7482..b298782 100644 --- a/src-tauri/src/services/waveform_generator.rs +++ b/src-tauri/src/services/waveform_generator.rs @@ -1 +1,90 @@ -// Implemented in Task 10 +use std::process::Command; + +pub fn extract_waveform(file_path: &str, sample_count: usize) -> Result, String> { + let output = Command::new("ffmpeg") + .args([ + "-i", + file_path, + "-ac", + "1", + "-filter:a", + &format!("aresample={sample_count}"), + "-f", + "f32le", + "-vn", + "-", + ]) + .output() + .map_err(|e| format!("Failed to run ffmpeg: {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("ffmpeg waveform extraction failed: {stderr}")); + } + + let samples = parse_f32_samples(&output.stdout); + let peaks = compute_peaks(&samples, sample_count); + Ok(peaks) +} + +fn parse_f32_samples(data: &[u8]) -> Vec { + data.chunks_exact(4) + .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) + .collect() +} + +fn compute_peaks(samples: &[f32], target_count: usize) -> Vec { + if samples.is_empty() || target_count == 0 { + return vec![0.0; target_count]; + } + + let chunk_size = (samples.len() as f64 / target_count as f64).ceil() as usize; + let chunk_size = chunk_size.max(1); + + let mut peaks: Vec = samples + .chunks(chunk_size) + .map(|chunk| chunk.iter().map(|s| s.abs() as f64).fold(0.0_f64, f64::max)) + .collect(); + + let max_peak = peaks.iter().cloned().fold(0.0_f64, f64::max); + if max_peak > 0.0 { + for peak in &mut peaks { + *peak /= max_peak; + } + } + + peaks.resize(target_count, 0.0); + peaks +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_compute_peaks_basic() { + let samples = vec![0.5, -0.8, 0.3, -0.1, 0.9, -0.2]; + let peaks = compute_peaks(&samples, 3); + assert_eq!(peaks.len(), 3); + assert!(peaks[0] > 0.0); + assert!(peaks[1] > 0.0); + assert!(peaks[2] > 0.0); + assert!((peaks[2] - 1.0).abs() < 0.001); + } + + #[test] + fn test_compute_peaks_empty() { + let peaks = compute_peaks(&[], 5); + assert_eq!(peaks.len(), 5); + assert!(peaks.iter().all(|p| *p == 0.0)); + } + + #[test] + fn test_parse_f32_samples() { + let val: f32 = 0.5; + let bytes = val.to_le_bytes(); + let samples = parse_f32_samples(&bytes); + assert_eq!(samples.len(), 1); + assert!((samples[0] - 0.5).abs() < 0.001); + } +} diff --git a/src/lib/bindings/mediaAnalysis.ts b/src/lib/bindings/mediaAnalysis.ts index c2ee2e6..1d21852 100644 --- a/src/lib/bindings/mediaAnalysis.ts +++ b/src/lib/bindings/mediaAnalysis.ts @@ -13,3 +13,10 @@ export interface ThumbnailSpritesheet { export async function extractKeyframes(filePath: string): Promise { return invoke('extract_keyframes', { filePath }); } + +export async function extractWaveform( + filePath: string, + sampleCount: number +): Promise { + return invoke('extract_waveform', { filePath, sampleCount }); +} diff --git a/src/lib/components/Timeline.svelte b/src/lib/components/Timeline.svelte index 77c5d6f..79d6647 100644 --- a/src/lib/components/Timeline.svelte +++ b/src/lib/components/Timeline.svelte @@ -69,7 +69,9 @@ session.duration, clipStore.clips, clipStore.selectedClipId, - clipStore.pendingInPoint + clipStore.pendingInPoint, + session.waveformPeaks, + session.thumbnailSpritesheets ); ctx.restore(); animFrameId = requestAnimationFrame(draw); diff --git a/src/lib/stores/videoSession.svelte.ts b/src/lib/stores/videoSession.svelte.ts index 472c6f0..ca74972 100644 --- a/src/lib/stores/videoSession.svelte.ts +++ b/src/lib/stores/videoSession.svelte.ts @@ -1,7 +1,7 @@ import type { VideoMetadata } from '$lib/bindings/video'; import { startDownload } from '$lib/bindings/video'; import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis'; -import { extractKeyframes } from '$lib/bindings/mediaAnalysis'; +import { extractKeyframes, extractWaveform } from '$lib/bindings/mediaAnalysis'; export type SessionStatus = 'idle' | 'resolving' | 'ready' | 'error'; export type DownloadStatus = 'idle' | 'downloading' | 'complete' | 'failed'; @@ -79,12 +79,22 @@ export function reset() { } async function triggerPostDownloadProcessing(filePath: string) { - try { - session.keyframePositions = await extractKeyframes(filePath); - } catch (e) { - console.error('Keyframe extraction failed:', e); + const [keyframes, waveform] = await Promise.allSettled([ + extractKeyframes(filePath), + extractWaveform(filePath, 8000), + ]); + + if (keyframes.status === 'fulfilled') { + session.keyframePositions = keyframes.value; + } else { + console.error('Keyframe extraction failed:', keyframes.reason); + } + + if (waveform.status === 'fulfilled') { + session.waveformPeaks = waveform.value; + } else { + console.error('Waveform extraction failed:', waveform.reason); } - // Waveform and thumbnails will be added in Tasks 10 and 11 } export async function beginDownload() { diff --git a/src/lib/timeline/renderer.ts b/src/lib/timeline/renderer.ts index 6ad9a2b..6104415 100644 --- a/src/lib/timeline/renderer.ts +++ b/src/lib/timeline/renderer.ts @@ -1,5 +1,7 @@ import { drawClips } from './clipRenderer'; import type { Clip } from '$lib/stores/clips.svelte'; +import { drawWaveform } from './waveformRenderer'; +import type { ThumbnailSpritesheet } from '$lib/bindings/mediaAnalysis'; export interface TimelineState { visibleStart: number; @@ -32,7 +34,9 @@ export function drawTimeline( duration: number, clips: Clip[] = [], selectedClipId: string | null = null, - pendingInPoint: number | null = null + pendingInPoint: number | null = null, + waveformPeaks: number[] = [], + _thumbnailSpritesheets: ThumbnailSpritesheet[] = [] ): void { const { width, height } = state; ctx.clearRect(0, 0, width, height); @@ -43,7 +47,12 @@ export function drawTimeline( drawTimeTicks(ctx, state, duration); drawPlaceholderLane(ctx, state, 0, THUMB_LANE_HEIGHT, 'Thumbnails'); - drawPlaceholderLane(ctx, state, THUMB_LANE_HEIGHT, WAVEFORM_LANE_HEIGHT, 'Waveform'); + + if (waveformPeaks.length > 0) { + drawWaveform(ctx, state, waveformPeaks, duration, THUMB_LANE_HEIGHT, WAVEFORM_LANE_HEIGHT); + } else { + drawPlaceholderLane(ctx, state, THUMB_LANE_HEIGHT, WAVEFORM_LANE_HEIGHT, 'Waveform'); + } drawClips(ctx, state, clips, selectedClipId, pendingInPoint); diff --git a/src/lib/timeline/waveformRenderer.ts b/src/lib/timeline/waveformRenderer.ts new file mode 100644 index 0000000..78ffd9b --- /dev/null +++ b/src/lib/timeline/waveformRenderer.ts @@ -0,0 +1,44 @@ +import type { TimelineState } from './renderer'; + +export function drawWaveform( + ctx: CanvasRenderingContext2D, + state: TimelineState, + peaks: number[], + duration: number, + y: number, + height: number +): void { + if (peaks.length === 0 || duration <= 0) return; + + const { width, visibleStart, visibleEnd } = state; + const visibleRange = visibleEnd - visibleStart; + + ctx.fillStyle = '#89b4fa44'; + ctx.strokeStyle = '#89b4fa88'; + ctx.lineWidth = 1; + + ctx.beginPath(); + ctx.moveTo(0, y + height); + + for (let px = 0; px < width; px++) { + const startSample = Math.floor( + ((visibleStart + ((px - 0.5) / width) * visibleRange) / duration) * peaks.length + ); + const endSample = Math.floor( + ((visibleStart + ((px + 0.5) / width) * visibleRange) / duration) * peaks.length + ); + + let maxPeak = 0; + for (let i = Math.max(0, startSample); i <= Math.min(endSample, peaks.length - 1); i++) { + maxPeak = Math.max(maxPeak, peaks[i]); + } + + const barHeight = maxPeak * height; + ctx.lineTo(px, y + height - barHeight); + } + + ctx.lineTo(width, y + height); + ctx.closePath(); + ctx.fill(); + ctx.stroke(); +}