diff --git a/crates/align-decode/src/apple.rs b/crates/align-decode/src/apple.rs index 8aa5c86..ad92288 100644 --- a/crates/align-decode/src/apple.rs +++ b/crates/align-decode/src/apple.rs @@ -503,6 +503,7 @@ impl MediaBackend for AppleNativeBackend { channels, bit_depth, is_float, + decoder_delay: 0.0, }); } let video = video_probe(&asset)?; diff --git a/crates/align-decode/src/backend.rs b/crates/align-decode/src/backend.rs index da83e81..7568267 100644 --- a/crates/align-decode/src/backend.rs +++ b/crates/align-decode/src/backend.rs @@ -76,6 +76,11 @@ pub struct AudioStreamProbe { pub channels: usize, pub bit_depth: Option, pub is_float: Option, + /// Seconds of decoder output before the stream's audio starts, which the + /// FFmpeg tier trims. Nonzero only for AC-3 and E-AC-3 in MPEG-TS + /// (AVCHD): their 256-sample codec delay is recorded nowhere, and + /// AVFoundation trims it, so both engines place the audio alike. + pub decoder_delay: f64, } #[derive(Clone, Debug, PartialEq)] diff --git a/crates/align-decode/src/export.rs b/crates/align-decode/src/export.rs index 0605510..7cbb6b1 100644 --- a/crates/align-decode/src/export.rs +++ b/crates/align-decode/src/export.rs @@ -1031,6 +1031,13 @@ fn export_media_file( } let ffmpeg = crate::ff::ffmpeg_bin() .ok_or_else(|| ExportError::Media("ffmpeg not found in PATH.".into()))?; + // FFmpeg decodes the recorder here, so trim the decoder delay its output + // keeps (AC-3 in MPEG-TS), as the backends do. + let decoder_delay = crate::ff::inspect(&plan.recorder_url) + .map_err(|error| ExportError::Media(format!("{}: {error}", plan.recorder_url.display())))? + .audio_streams + .first() + .map_or(0.0, |stream| stream.decoder_delay); let delay_samples = (plan.leading_silence.max(0.0) * 48_000.0).round() as u64; let filter = format!( "[1:a:0]atrim=duration={:.9},asetpts=PTS-STARTPTS,aresample=48000,adelay={delay_samples}S:all=1,apad=whole_dur={:.9},atrim=duration={:.9}[clean]", @@ -1039,7 +1046,11 @@ fn export_media_file( let mut child = crate::ff::command(ffmpeg) .args(["-y", "-v", "error", "-i"]) .arg(&plan.video_url) - .args(["-ss", &format!("{:.9}", plan.recorder_source_in), "-i"]) + .args([ + "-ss", + &format!("{:.9}", plan.recorder_source_in + decoder_delay), + "-i", + ]) .arg(&plan.recorder_url) .args(["-filter_complex", &filter]) .args(["-map", "0:v:0", "-map", "[clean]"]) @@ -1677,4 +1688,56 @@ mod tests { assert!((24_000..24_100).contains(&first_signal)); let _ = std::fs::remove_dir_all(&dir); } + + /// FFmpeg remuxes cameras outside QuickTime and MP4 and then decodes the + /// recorder itself. It must trim the recorder's AC-3 decoder delay as the + /// backends do, or the clean audio lands 256 samples late. + #[test] + fn ffmpeg_remux_trims_the_recorder_decoder_delay() { + let Some(ffmpeg) = crate::ff::ffmpeg_bin() else { + eprintln!("SKIP: ffmpeg is unavailable"); + return; + }; + let dir = tempfile::tempdir().unwrap(); + let video = dir.path().join("camera.mts"); + let recorder = dir.path().join("recorder.mts"); + let video_ok = crate::ff::command(&ffmpeg) + .args(["-y", "-v", "error", "-f", "lavfi", "-i"]) + .arg("color=c=black:s=160x90:r=25:d=2") + .args(["-c:v", "mpeg2video", "-f", "mpegts"]) + .arg(&video) + .status() + .is_ok_and(|status| status.success()); + if !video_ok || !crate::ff::write_ac3_transport_stream(&recorder, 0.25, 1.0) { + eprintln!("SKIP: fixture generation is unsupported by this ffmpeg"); + return; + } + let plan = MediaFilePlan { + video_url: video, + recorder_url: recorder, + recorder_source_in: 0.1, + leading_silence: 0.5, + recorder_duration: 0.9, + video_duration: 2.0, + url: dir.path().join("out.mov"), + }; + + export_media_file( + &crate::portable::PortableBackend, + &plan, + &std::sync::atomic::AtomicBool::new(false), + ) + .unwrap(); + + let mut left = Vec::new(); + crate::portable::PortableBackend + .decode_native(&plan.url, 0, None, &mut |block| { + left.extend_from_slice(&block.frames[0]); + Ok(()) + }) + .unwrap(); + // 0.5 s of leading silence, then the click 0.15 s into the recorder. + let at = crate::ff::click_at(&left); + assert!(at.abs_diff(31_200) <= 2, "click at {at}"); + } } diff --git a/crates/align-decode/src/ff.rs b/crates/align-decode/src/ff.rs index 5fb6455..53d6ae3 100644 --- a/crates/align-decode/src/ff.rs +++ b/crates/align-decode/src/ff.rs @@ -115,6 +115,13 @@ fn num(value: &serde_json::Value) -> Option { pub fn parse_ffprobe_json(json: &[u8]) -> Option { let v: serde_json::Value = serde_json::from_slice(json).ok()?; let streams = v.get("streams")?.as_array()?; + // MPEG-TS records no edit list, so FFmpeg's output keeps AC-3's codec + // delay; in MP4 and MOV FFmpeg trims it through the edit list. + let transport_stream = v + .get("format") + .and_then(|f| f.get("format_name")) + .and_then(|n| n.as_str()) + .is_some_and(|n| n.split(',').any(|n| n == "mpegts")); let mut audio_streams = Vec::new(); let mut has_video = false; for s in streams { @@ -128,14 +135,21 @@ pub fn parse_ffprobe_json(json: &[u8]) -> Option { } else { None }; + let sample_rate = num(s.get("sample_rate")?)?; + let decoder_delay = if transport_stream && matches!(codec, "ac3" | "eac3") { + 256.0 / sample_rate + } else { + 0.0 + }; audio_streams.push(AudioStreamProbe { - sample_rate: num(s.get("sample_rate")?)?, + sample_rate, channels: s.get("channels")?.as_u64()? as usize, bit_depth: s .get("bits_per_sample") .and_then(|b| b.as_u64()) .map(|b| b as u32), is_float, + decoder_delay, }) } "video" => { @@ -612,6 +626,9 @@ impl Drop for AudioPipeI32 { } } +/// A window duration that reaches the end of any file. +pub(crate) const WHOLE_FILE: f64 = 1e9; + /// Full-file fallback decode to 8 kHz mono (fingerprint path). pub fn decode_mono_8k( path: &Path, @@ -623,15 +640,15 @@ pub fn decode_mono_8k( ) -> Result<(), DecodeError> { // Native rate unknown here — inspect first (header-only, cheap). let probe = inspect(path)?; - let sample_rate = probe + let (sample_rate, delay) = probe .audio_streams .get(stream_index) - .map(|s| s.sample_rate) - .unwrap_or(0.0); + .map_or((0.0, 0.0), |s| (s.sample_rate, s.decoder_delay)); if sample_rate <= 0.0 { return Err(DecodeError::InvalidPcm); } - let mut pipe = AudioPipe::spawn(path, stream_index, None, channels)?; + let window = (delay > 0.0).then_some((delay, WHOLE_FILE)); + let mut pipe = AudioPipe::spawn(path, stream_index, window, channels)?; let mut mono = crate::mono::MonoPipe::new( sample_rate, 8000.0, @@ -660,17 +677,21 @@ pub fn decode_window( mix_channels: bool, ) -> Result<(f64, Vec), DecodeError> { let probe = inspect(path)?; - let sample_rate = probe + let (sample_rate, delay) = probe .audio_streams .get(stream_index) - .map(|s| s.sample_rate) - .unwrap_or(0.0); + .map_or((0.0, 0.0), |s| (s.sample_rate, s.decoder_delay)); if sample_rate <= 0.0 { return Err(DecodeError::InvalidPcm); } let actual_start = start.max(0.0); let mut samples = Vec::new(); - let mut pipe = AudioPipe::spawn(path, stream_index, Some((actual_start, duration)), channels)?; + let mut pipe = AudioPipe::spawn( + path, + stream_index, + Some((actual_start + delay, duration)), + channels, + )?; let mut mono = crate::mono::MonoPipe::new( sample_rate, 16_000.0, @@ -692,8 +713,67 @@ pub fn decode_window( Ok((actual_start, samples)) } +/// Write stereo 48 kHz AC-3 in MPEG-TS, silent except for a 2 ms click at +/// `click` seconds. False (the test skips) without an AC-3 encoder. +#[cfg(test)] +pub(crate) fn write_ac3_transport_stream(path: &Path, click: f64, seconds: f64) -> bool { + let Some(ffmpeg) = ffmpeg_bin() else { + return false; + }; + let source = format!( + "aevalsrc='if(between(t,{click},{}),0.9,0)':s=48000:d={seconds}", + click + 0.002 + ); + let ok = command(&ffmpeg) + .args(["-v", "error", "-y", "-f", "lavfi", "-i", &source]) + .args(["-ac", "2", "-c:a", "ac3", "-b:a", "448k", "-f", "mpegts"]) + .arg(path) + .status() + .is_ok_and(|s| s.success()); + if !ok { + eprintln!("SKIP: no AC-3 encoder"); + } + ok +} + +/// Index of the first sample of a `write_ac3_transport_stream` click. +#[cfg(test)] +pub(crate) fn click_at(samples: &[f32]) -> usize { + samples.iter().position(|s| s.abs() > 0.3).unwrap() +} + #[cfg(test)] mod tests { + /// AC-3 decodes 256 samples late, and MPEG-TS records it nowhere (MP4 + /// and MOV use an edit list, which FFmpeg applies). AVCHD audio must sit + /// where AVFoundation places it: on its timeline, not 5.3 ms early. + #[test] + fn ac3_in_transport_streams_starts_on_time() { + use crate::backend::MediaBackend; + let dir = tempfile::tempdir().unwrap(); + let clip = dir.path().join("00001.MTS"); + if !write_ac3_transport_stream(&clip, 1.5, 4.0) { + return; + } + let backend = crate::portable::PortableBackend; + for start in [0.0, 0.7] { + let mut left = Vec::new(); + let actual = backend + .decode_native(&clip, 0, Some((start, None)), &mut |block| { + left.extend_from_slice(&block.frames[0]); + Ok(()) + }) + .unwrap(); + let at = click_at(&left) + (actual * 48_000.0).round() as usize; + assert!(at.abs_diff(72_000) <= 2, "from {start} s: click at {at}"); + } + let (window_start, window) = backend + .decode_window_16k(&clip, 1.0, 1.0, align_core::AudioAnalysisSource::default()) + .unwrap(); + let at = window_start * 1000.0 + click_at(&window) as f64 / 16.0; + assert!((at - 1500.0).abs() < 0.3, "window click at {at} ms"); + } + #[test] fn pipe_fragments_preserve_complete_multichannel_frames() { for channels in [1, 2, 3, 6, 8] { diff --git a/crates/align-decode/src/isobmff.rs b/crates/align-decode/src/isobmff.rs index 0d04e9a..57846c1 100644 --- a/crates/align-decode/src/isobmff.rs +++ b/crates/align-decode/src/isobmff.rs @@ -327,6 +327,7 @@ impl Movie { channels: a.channels, bit_depth: a.bit_depth, is_float: a.is_float, + decoder_delay: 0.0, }) .collect(); let video = self.video().map(|track| VideoProbe { diff --git a/crates/align-decode/src/pcm.rs b/crates/align-decode/src/pcm.rs index 17bba36..f824829 100644 --- a/crates/align-decode/src/pcm.rs +++ b/crates/align-decode/src/pcm.rs @@ -193,6 +193,7 @@ impl PcmTrack { channels: self.channels, bit_depth: Some(self.bits), is_float: Some(self.float), + decoder_delay: 0.0, }], has_video: false, video: None, diff --git a/crates/align-decode/src/portable.rs b/crates/align-decode/src/portable.rs index deb64b1..241133b 100644 --- a/crates/align-decode/src/portable.rs +++ b/crates/align-decode/src/portable.rs @@ -71,7 +71,10 @@ impl MediaBackend for PortableBackend { let mut pipe = crate::ff::AudioPipe::spawn( path, stream_index, - Some((actual, duration.unwrap_or(1e9))), + Some(( + actual + stream.decoder_delay, + duration.unwrap_or(crate::ff::WHOLE_FILE), + )), channels, )?; let ok = pipe.pump(&mut |frames| { @@ -122,7 +125,10 @@ impl MediaBackend for PortableBackend { let mut pipe = crate::ff::AudioPipeI32::spawn( path, stream_index, - Some((actual, duration.unwrap_or(1e9))), + Some(( + actual + stream.decoder_delay, + duration.unwrap_or(crate::ff::WHOLE_FILE), + )), channels, )?; let ok = pipe.pump(&mut |frames| { diff --git a/crates/align-decode/src/render.rs b/crates/align-decode/src/render.rs index 9bd2a1f..2c350de 100644 --- a/crates/align-decode/src/render.rs +++ b/crates/align-decode/src/render.rs @@ -77,10 +77,12 @@ pub(crate) fn check_cancel(cancel: &AtomicBool) -> Result<(), RenderError> { } } -/// A camera's last audio sample may precede its last video frame. After a -/// successful decode, fill only that sub-frame gap at the container's end. -/// Never conceal truncated audio-only files or missing audio inside an edit. -fn finish_video_tail( +/// A camera's last audio sample may precede its last video frame, and audio +/// whose decoder delay is trimmed from its start (AC-3 in MPEG-TS) ends that +/// many samples before its container. After a successful decode, fill only +/// those gaps at the container's end. Never conceal truncated audio or +/// missing audio inside an edit. +fn finish_tail( writer: &mut WavWriter, probe: &crate::backend::ProbeReport, end_seconds: f64, @@ -91,14 +93,22 @@ fn finish_video_tail( if remaining == 0 { return Ok(()); } - let frame_duration = probe - .video - .as_ref() - .and_then(|v| v.frame_duration) - .map_or(0.001, |t| t.as_seconds()); - if !probe.has_video - || (end_seconds - probe.duration_seconds).abs() > 1.0 / sample_rate - || remaining as f64 > (frame_duration * sample_rate).ceil() + let delay = probe + .audio_streams + .first() + .map_or(0.0, |s| (s.decoder_delay * sample_rate).round()); + let frame = if probe.has_video { + let frame_duration = probe + .video + .as_ref() + .and_then(|v| v.frame_duration) + .map_or(0.001, |t| t.as_seconds()); + (frame_duration * sample_rate).ceil() + } else { + 0.0 + }; + if (end_seconds - probe.duration_seconds).abs() > 1.0 / sample_rate + || remaining as f64 > delay + frame { return Err(RenderError::CannotRead); } @@ -668,7 +678,7 @@ pub fn render_pad_cancellable( }) .map_err(RenderError::from)?; if let (Some(n), Some((_, end))) = (remaining, selection) { - finish_video_tail( + finish_tail( &mut writer, &probe, end as f64 / sample_rate, @@ -796,7 +806,7 @@ pub fn render_channel_cancellable( ) .map_err(RenderError::from)?; } - finish_video_tail( + finish_tail( &mut writer, &probe, source_start.max(0.0) + duration, @@ -1144,24 +1154,59 @@ mod tests { let dst = dir.join("tail.wav"); let mut writer = WavWriter::create(&dst, 48000, 2, false).unwrap(); assert_eq!( - finish_video_tail(&mut writer, &probe, 1.0, 6, 48000.0, 2), + finish_tail(&mut writer, &probe, 1.0, 6, 48000.0, 2), Err(RenderError::CannotRead) ); probe.has_video = true; assert_eq!( - finish_video_tail(&mut writer, &probe, 0.9, 6, 48000.0, 2), + finish_tail(&mut writer, &probe, 0.9, 6, 48000.0, 2), Err(RenderError::CannotRead) ); assert_eq!( - finish_video_tail(&mut writer, &probe, 1.0, 4800, 48000.0, 2), + finish_tail(&mut writer, &probe, 1.0, 4800, 48000.0, 2), Err(RenderError::CannotRead) ); - finish_video_tail(&mut writer, &probe, 1.0, 6, 48000.0, 2).unwrap(); + finish_tail(&mut writer, &probe, 1.0, 6, 48000.0, 2).unwrap(); writer.finalize().unwrap(); assert_eq!(read_pad_samples(&dst), vec![0.0; 12]); std::fs::remove_dir_all(dir).unwrap(); } + /// AC-3 in MPEG-TS decodes 256 samples late. Trimming them leaves the + /// container's last 256 samples without audio, yet a stem of the whole + /// file keeps the container's length and its audio stays on time. + #[test] + fn stems_of_audio_with_a_trimmed_decoder_delay_keep_their_length() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("recorder.mts"); + if !crate::ff::write_ac3_transport_stream(&source, 0.5, 1.0) { + return; + } + let backend = crate::portable::PortableBackend; + let probe = backend.inspect(&source).unwrap(); + assert!(!probe.has_video); + let stem = dir.path().join("stem.wav"); + render_channel_cancellable( + &backend, + &source, + &stem, + 0, + None, + false, + 0.0, + probe.duration_seconds, + 0, + &NEVER_CANCELLED, + ) + .unwrap(); + let samples = read_pad_samples(&stem); + assert_eq!( + samples.len(), + (probe.duration_seconds * 48_000.0).round() as usize + ); + assert!(crate::ff::click_at(&samples).abs_diff(24_000) <= 2); + } + #[test] fn placement_pad_invalid_clock_preserves_existing_destination() { let dir = tmp("pad-clock-underflow"); diff --git a/crates/align-decode/src/sym.rs b/crates/align-decode/src/sym.rs index 2a6a8f3..19fdcb2 100644 --- a/crates/align-decode/src/sym.rs +++ b/crates/align-decode/src/sym.rs @@ -144,6 +144,7 @@ pub fn inspect(path: &Path) -> Result { channels: p.channels.map(|c| c.count()).unwrap_or(0), bit_depth: bits.map(|(b, _)| b), is_float: bits.map(|(_, f)| f), + decoder_delay: 0.0, }); if duration_seconds.is_none() { if let (Some(n), Some(rate)) = (p.n_frames, p.sample_rate) {