From 601830ed4060d4fe914525577aa2572cb0217444 Mon Sep 17 00:00:00 2001 From: FedyaLight Date: Thu, 12 Nov 2026 01:12:00 +0300 Subject: [PATCH 1/2] Trim the AC-3 decoder delay in transport streams AC-3 and E-AC-3 decode 256 samples late. MP4 and MOV record that in their edit list, which FFmpeg applies, but MPEG-TS (AVCHD) records it nowhere, so on Windows and Linux AVCHD audio sat 5.3 ms earlier than on macOS, where AVFoundation trims the delay. The FFmpeg tier now starts those streams 256 samples later for whole-file, window and native decoding. On camera-original Sony HDR-CX6 and AVCHD Lite clips the portable engine now places the audio exactly where AVFoundation does. --- crates/align-decode/src/apple.rs | 1 + crates/align-decode/src/backend.rs | 5 ++ crates/align-decode/src/ff.rs | 81 +++++++++++++++++++++++++---- crates/align-decode/src/isobmff.rs | 1 + crates/align-decode/src/pcm.rs | 1 + crates/align-decode/src/portable.rs | 10 +++- crates/align-decode/src/sym.rs | 1 + 7 files changed, 89 insertions(+), 11 deletions(-) diff --git a/crates/align-decode/src/apple.rs b/crates/align-decode/src/apple.rs index 8aa5c86..ad92288 100644 --- a/crates/align-decode/src/apple.rs +++ b/crates/align-decode/src/apple.rs @@ -503,6 +503,7 @@ impl MediaBackend for AppleNativeBackend { channels, bit_depth, is_float, + decoder_delay: 0.0, }); } let video = video_probe(&asset)?; diff --git a/crates/align-decode/src/backend.rs b/crates/align-decode/src/backend.rs index da83e81..7568267 100644 --- a/crates/align-decode/src/backend.rs +++ b/crates/align-decode/src/backend.rs @@ -76,6 +76,11 @@ pub struct AudioStreamProbe { pub channels: usize, pub bit_depth: Option, pub is_float: Option, + /// Seconds of decoder output before the stream's audio starts, which the + /// FFmpeg tier trims. Nonzero only for AC-3 and E-AC-3 in MPEG-TS + /// (AVCHD): their 256-sample codec delay is recorded nowhere, and + /// AVFoundation trims it, so both engines place the audio alike. + pub decoder_delay: f64, } #[derive(Clone, Debug, PartialEq)] diff --git a/crates/align-decode/src/ff.rs b/crates/align-decode/src/ff.rs index 5fb6455..956f5dd 100644 --- a/crates/align-decode/src/ff.rs +++ b/crates/align-decode/src/ff.rs @@ -115,6 +115,13 @@ fn num(value: &serde_json::Value) -> Option { pub fn parse_ffprobe_json(json: &[u8]) -> Option { let v: serde_json::Value = serde_json::from_slice(json).ok()?; let streams = v.get("streams")?.as_array()?; + // MPEG-TS records no edit list, so FFmpeg's output keeps AC-3's codec + // delay; in MP4 and MOV FFmpeg trims it through the edit list. + let transport_stream = v + .get("format") + .and_then(|f| f.get("format_name")) + .and_then(|n| n.as_str()) + .is_some_and(|n| n.split(',').any(|n| n == "mpegts")); let mut audio_streams = Vec::new(); let mut has_video = false; for s in streams { @@ -128,14 +135,21 @@ pub fn parse_ffprobe_json(json: &[u8]) -> Option { } else { None }; + let sample_rate = num(s.get("sample_rate")?)?; + let decoder_delay = if transport_stream && matches!(codec, "ac3" | "eac3") { + 256.0 / sample_rate + } else { + 0.0 + }; audio_streams.push(AudioStreamProbe { - sample_rate: num(s.get("sample_rate")?)?, + sample_rate, channels: s.get("channels")?.as_u64()? as usize, bit_depth: s .get("bits_per_sample") .and_then(|b| b.as_u64()) .map(|b| b as u32), is_float, + decoder_delay, }) } "video" => { @@ -612,6 +626,9 @@ impl Drop for AudioPipeI32 { } } +/// A window duration that reaches the end of any file. +pub(crate) const WHOLE_FILE: f64 = 1e9; + /// Full-file fallback decode to 8 kHz mono (fingerprint path). pub fn decode_mono_8k( path: &Path, @@ -623,15 +640,15 @@ pub fn decode_mono_8k( ) -> Result<(), DecodeError> { // Native rate unknown here — inspect first (header-only, cheap). let probe = inspect(path)?; - let sample_rate = probe + let (sample_rate, delay) = probe .audio_streams .get(stream_index) - .map(|s| s.sample_rate) - .unwrap_or(0.0); + .map_or((0.0, 0.0), |s| (s.sample_rate, s.decoder_delay)); if sample_rate <= 0.0 { return Err(DecodeError::InvalidPcm); } - let mut pipe = AudioPipe::spawn(path, stream_index, None, channels)?; + let window = (delay > 0.0).then_some((delay, WHOLE_FILE)); + let mut pipe = AudioPipe::spawn(path, stream_index, window, channels)?; let mut mono = crate::mono::MonoPipe::new( sample_rate, 8000.0, @@ -660,17 +677,21 @@ pub fn decode_window( mix_channels: bool, ) -> Result<(f64, Vec), DecodeError> { let probe = inspect(path)?; - let sample_rate = probe + let (sample_rate, delay) = probe .audio_streams .get(stream_index) - .map(|s| s.sample_rate) - .unwrap_or(0.0); + .map_or((0.0, 0.0), |s| (s.sample_rate, s.decoder_delay)); if sample_rate <= 0.0 { return Err(DecodeError::InvalidPcm); } let actual_start = start.max(0.0); let mut samples = Vec::new(); - let mut pipe = AudioPipe::spawn(path, stream_index, Some((actual_start, duration)), channels)?; + let mut pipe = AudioPipe::spawn( + path, + stream_index, + Some((actual_start + delay, duration)), + channels, + )?; let mut mono = crate::mono::MonoPipe::new( sample_rate, 16_000.0, @@ -694,6 +715,48 @@ pub fn decode_window( #[cfg(test)] mod tests { + /// AC-3 decodes 256 samples late, and MPEG-TS records it nowhere (MP4 + /// and MOV use an edit list, which FFmpeg applies). AVCHD audio must sit + /// where AVFoundation places it: on its timeline, not 5.3 ms early. + #[test] + fn ac3_in_transport_streams_starts_on_time() { + use crate::backend::MediaBackend; + let Some(ffmpeg) = ffmpeg_bin() else { + return; + }; + let dir = tempfile::tempdir().unwrap(); + let clip = dir.path().join("00001.MTS"); + let ok = command(&ffmpeg) + .args(["-v", "error", "-y", "-f", "lavfi", "-i"]) + .arg("aevalsrc='if(between(t,1.5,1.502),0.9,0)':s=48000:d=4") + .args(["-ac", "2", "-c:a", "ac3", "-b:a", "448k", "-f", "mpegts"]) + .arg(&clip) + .status() + .is_ok_and(|s| s.success()); + if !ok { + eprintln!("SKIP: no AC-3 encoder"); + return; + } + let backend = crate::portable::PortableBackend; + let click = |samples: &[f32]| samples.iter().position(|s| s.abs() > 0.3).unwrap(); + for start in [0.0, 0.7] { + let mut left = Vec::new(); + let actual = backend + .decode_native(&clip, 0, Some((start, None)), &mut |block| { + left.extend_from_slice(&block.frames[0]); + Ok(()) + }) + .unwrap(); + let at = click(&left) + (actual * 48_000.0).round() as usize; + assert!(at.abs_diff(72_000) <= 2, "from {start} s: click at {at}"); + } + let (window_start, window) = backend + .decode_window_16k(&clip, 1.0, 1.0, align_core::AudioAnalysisSource::default()) + .unwrap(); + let at = window_start * 1000.0 + click(&window) as f64 / 16.0; + assert!((at - 1500.0).abs() < 0.3, "window click at {at} ms"); + } + #[test] fn pipe_fragments_preserve_complete_multichannel_frames() { for channels in [1, 2, 3, 6, 8] { diff --git a/crates/align-decode/src/isobmff.rs b/crates/align-decode/src/isobmff.rs index 0d04e9a..57846c1 100644 --- a/crates/align-decode/src/isobmff.rs +++ b/crates/align-decode/src/isobmff.rs @@ -327,6 +327,7 @@ impl Movie { channels: a.channels, bit_depth: a.bit_depth, is_float: a.is_float, + decoder_delay: 0.0, }) .collect(); let video = self.video().map(|track| VideoProbe { diff --git a/crates/align-decode/src/pcm.rs b/crates/align-decode/src/pcm.rs index 17bba36..f824829 100644 --- a/crates/align-decode/src/pcm.rs +++ b/crates/align-decode/src/pcm.rs @@ -193,6 +193,7 @@ impl PcmTrack { channels: self.channels, bit_depth: Some(self.bits), is_float: Some(self.float), + decoder_delay: 0.0, }], has_video: false, video: None, diff --git a/crates/align-decode/src/portable.rs b/crates/align-decode/src/portable.rs index deb64b1..241133b 100644 --- a/crates/align-decode/src/portable.rs +++ b/crates/align-decode/src/portable.rs @@ -71,7 +71,10 @@ impl MediaBackend for PortableBackend { let mut pipe = crate::ff::AudioPipe::spawn( path, stream_index, - Some((actual, duration.unwrap_or(1e9))), + Some(( + actual + stream.decoder_delay, + duration.unwrap_or(crate::ff::WHOLE_FILE), + )), channels, )?; let ok = pipe.pump(&mut |frames| { @@ -122,7 +125,10 @@ impl MediaBackend for PortableBackend { let mut pipe = crate::ff::AudioPipeI32::spawn( path, stream_index, - Some((actual, duration.unwrap_or(1e9))), + Some(( + actual + stream.decoder_delay, + duration.unwrap_or(crate::ff::WHOLE_FILE), + )), channels, )?; let ok = pipe.pump(&mut |frames| { diff --git a/crates/align-decode/src/sym.rs b/crates/align-decode/src/sym.rs index 2a6a8f3..19fdcb2 100644 --- a/crates/align-decode/src/sym.rs +++ b/crates/align-decode/src/sym.rs @@ -144,6 +144,7 @@ pub fn inspect(path: &Path) -> Result { channels: p.channels.map(|c| c.count()).unwrap_or(0), bit_depth: bits.map(|(b, _)| b), is_float: bits.map(|(_, f)| f), + decoder_delay: 0.0, }); if duration_seconds.is_none() { if let (Some(n), Some(rate)) = (p.n_frames, p.sample_rate) { From 260d0848b480c71f08a6d885f0f8799c87cf8231 Mon Sep 17 00:00:00 2001 From: FedyaLight Date: Thu, 12 Nov 2026 02:02:21 +0300 Subject: [PATCH 2/2] Keep exports consistent with the trimmed AC-3 delay Trimming 256 samples from the start of AC-3 in MPEG-TS leaves the last 256 samples of the container without audio. Stems and placement pads now fill that gap with silence, as they fill a camera's sub-frame tail, so an audio-only transport stream exports at its container length instead of failing with "Cannot decode audio for drift correction." Clean-media export through FFmpeg (cameras outside QuickTime and MP4) decodes the recorder with FFmpeg directly; it now skips the recorder's decoder delay too, instead of placing its audio 256 samples late. --- crates/align-decode/src/export.rs | 65 ++++++++++++++++++++++++- crates/align-decode/src/ff.rs | 47 ++++++++++++------ crates/align-decode/src/render.rs | 81 ++++++++++++++++++++++++------- 3 files changed, 159 insertions(+), 34 deletions(-) diff --git a/crates/align-decode/src/export.rs b/crates/align-decode/src/export.rs index 0605510..7cbb6b1 100644 --- a/crates/align-decode/src/export.rs +++ b/crates/align-decode/src/export.rs @@ -1031,6 +1031,13 @@ fn export_media_file( } let ffmpeg = crate::ff::ffmpeg_bin() .ok_or_else(|| ExportError::Media("ffmpeg not found in PATH.".into()))?; + // FFmpeg decodes the recorder here, so trim the decoder delay its output + // keeps (AC-3 in MPEG-TS), as the backends do. + let decoder_delay = crate::ff::inspect(&plan.recorder_url) + .map_err(|error| ExportError::Media(format!("{}: {error}", plan.recorder_url.display())))? + .audio_streams + .first() + .map_or(0.0, |stream| stream.decoder_delay); let delay_samples = (plan.leading_silence.max(0.0) * 48_000.0).round() as u64; let filter = format!( "[1:a:0]atrim=duration={:.9},asetpts=PTS-STARTPTS,aresample=48000,adelay={delay_samples}S:all=1,apad=whole_dur={:.9},atrim=duration={:.9}[clean]", @@ -1039,7 +1046,11 @@ fn export_media_file( let mut child = crate::ff::command(ffmpeg) .args(["-y", "-v", "error", "-i"]) .arg(&plan.video_url) - .args(["-ss", &format!("{:.9}", plan.recorder_source_in), "-i"]) + .args([ + "-ss", + &format!("{:.9}", plan.recorder_source_in + decoder_delay), + "-i", + ]) .arg(&plan.recorder_url) .args(["-filter_complex", &filter]) .args(["-map", "0:v:0", "-map", "[clean]"]) @@ -1677,4 +1688,56 @@ mod tests { assert!((24_000..24_100).contains(&first_signal)); let _ = std::fs::remove_dir_all(&dir); } + + /// FFmpeg remuxes cameras outside QuickTime and MP4 and then decodes the + /// recorder itself. It must trim the recorder's AC-3 decoder delay as the + /// backends do, or the clean audio lands 256 samples late. + #[test] + fn ffmpeg_remux_trims_the_recorder_decoder_delay() { + let Some(ffmpeg) = crate::ff::ffmpeg_bin() else { + eprintln!("SKIP: ffmpeg is unavailable"); + return; + }; + let dir = tempfile::tempdir().unwrap(); + let video = dir.path().join("camera.mts"); + let recorder = dir.path().join("recorder.mts"); + let video_ok = crate::ff::command(&ffmpeg) + .args(["-y", "-v", "error", "-f", "lavfi", "-i"]) + .arg("color=c=black:s=160x90:r=25:d=2") + .args(["-c:v", "mpeg2video", "-f", "mpegts"]) + .arg(&video) + .status() + .is_ok_and(|status| status.success()); + if !video_ok || !crate::ff::write_ac3_transport_stream(&recorder, 0.25, 1.0) { + eprintln!("SKIP: fixture generation is unsupported by this ffmpeg"); + return; + } + let plan = MediaFilePlan { + video_url: video, + recorder_url: recorder, + recorder_source_in: 0.1, + leading_silence: 0.5, + recorder_duration: 0.9, + video_duration: 2.0, + url: dir.path().join("out.mov"), + }; + + export_media_file( + &crate::portable::PortableBackend, + &plan, + &std::sync::atomic::AtomicBool::new(false), + ) + .unwrap(); + + let mut left = Vec::new(); + crate::portable::PortableBackend + .decode_native(&plan.url, 0, None, &mut |block| { + left.extend_from_slice(&block.frames[0]); + Ok(()) + }) + .unwrap(); + // 0.5 s of leading silence, then the click 0.15 s into the recorder. + let at = crate::ff::click_at(&left); + assert!(at.abs_diff(31_200) <= 2, "click at {at}"); + } } diff --git a/crates/align-decode/src/ff.rs b/crates/align-decode/src/ff.rs index 956f5dd..53d6ae3 100644 --- a/crates/align-decode/src/ff.rs +++ b/crates/align-decode/src/ff.rs @@ -713,6 +713,35 @@ pub fn decode_window( Ok((actual_start, samples)) } +/// Write stereo 48 kHz AC-3 in MPEG-TS, silent except for a 2 ms click at +/// `click` seconds. False (the test skips) without an AC-3 encoder. +#[cfg(test)] +pub(crate) fn write_ac3_transport_stream(path: &Path, click: f64, seconds: f64) -> bool { + let Some(ffmpeg) = ffmpeg_bin() else { + return false; + }; + let source = format!( + "aevalsrc='if(between(t,{click},{}),0.9,0)':s=48000:d={seconds}", + click + 0.002 + ); + let ok = command(&ffmpeg) + .args(["-v", "error", "-y", "-f", "lavfi", "-i", &source]) + .args(["-ac", "2", "-c:a", "ac3", "-b:a", "448k", "-f", "mpegts"]) + .arg(path) + .status() + .is_ok_and(|s| s.success()); + if !ok { + eprintln!("SKIP: no AC-3 encoder"); + } + ok +} + +/// Index of the first sample of a `write_ac3_transport_stream` click. +#[cfg(test)] +pub(crate) fn click_at(samples: &[f32]) -> usize { + samples.iter().position(|s| s.abs() > 0.3).unwrap() +} + #[cfg(test)] mod tests { /// AC-3 decodes 256 samples late, and MPEG-TS records it nowhere (MP4 @@ -721,24 +750,12 @@ mod tests { #[test] fn ac3_in_transport_streams_starts_on_time() { use crate::backend::MediaBackend; - let Some(ffmpeg) = ffmpeg_bin() else { - return; - }; let dir = tempfile::tempdir().unwrap(); let clip = dir.path().join("00001.MTS"); - let ok = command(&ffmpeg) - .args(["-v", "error", "-y", "-f", "lavfi", "-i"]) - .arg("aevalsrc='if(between(t,1.5,1.502),0.9,0)':s=48000:d=4") - .args(["-ac", "2", "-c:a", "ac3", "-b:a", "448k", "-f", "mpegts"]) - .arg(&clip) - .status() - .is_ok_and(|s| s.success()); - if !ok { - eprintln!("SKIP: no AC-3 encoder"); + if !write_ac3_transport_stream(&clip, 1.5, 4.0) { return; } let backend = crate::portable::PortableBackend; - let click = |samples: &[f32]| samples.iter().position(|s| s.abs() > 0.3).unwrap(); for start in [0.0, 0.7] { let mut left = Vec::new(); let actual = backend @@ -747,13 +764,13 @@ mod tests { Ok(()) }) .unwrap(); - let at = click(&left) + (actual * 48_000.0).round() as usize; + let at = click_at(&left) + (actual * 48_000.0).round() as usize; assert!(at.abs_diff(72_000) <= 2, "from {start} s: click at {at}"); } let (window_start, window) = backend .decode_window_16k(&clip, 1.0, 1.0, align_core::AudioAnalysisSource::default()) .unwrap(); - let at = window_start * 1000.0 + click(&window) as f64 / 16.0; + let at = window_start * 1000.0 + click_at(&window) as f64 / 16.0; assert!((at - 1500.0).abs() < 0.3, "window click at {at} ms"); } diff --git a/crates/align-decode/src/render.rs b/crates/align-decode/src/render.rs index 9bd2a1f..2c350de 100644 --- a/crates/align-decode/src/render.rs +++ b/crates/align-decode/src/render.rs @@ -77,10 +77,12 @@ pub(crate) fn check_cancel(cancel: &AtomicBool) -> Result<(), RenderError> { } } -/// A camera's last audio sample may precede its last video frame. After a -/// successful decode, fill only that sub-frame gap at the container's end. -/// Never conceal truncated audio-only files or missing audio inside an edit. -fn finish_video_tail( +/// A camera's last audio sample may precede its last video frame, and audio +/// whose decoder delay is trimmed from its start (AC-3 in MPEG-TS) ends that +/// many samples before its container. After a successful decode, fill only +/// those gaps at the container's end. Never conceal truncated audio or +/// missing audio inside an edit. +fn finish_tail( writer: &mut WavWriter, probe: &crate::backend::ProbeReport, end_seconds: f64, @@ -91,14 +93,22 @@ fn finish_video_tail( if remaining == 0 { return Ok(()); } - let frame_duration = probe - .video - .as_ref() - .and_then(|v| v.frame_duration) - .map_or(0.001, |t| t.as_seconds()); - if !probe.has_video - || (end_seconds - probe.duration_seconds).abs() > 1.0 / sample_rate - || remaining as f64 > (frame_duration * sample_rate).ceil() + let delay = probe + .audio_streams + .first() + .map_or(0.0, |s| (s.decoder_delay * sample_rate).round()); + let frame = if probe.has_video { + let frame_duration = probe + .video + .as_ref() + .and_then(|v| v.frame_duration) + .map_or(0.001, |t| t.as_seconds()); + (frame_duration * sample_rate).ceil() + } else { + 0.0 + }; + if (end_seconds - probe.duration_seconds).abs() > 1.0 / sample_rate + || remaining as f64 > delay + frame { return Err(RenderError::CannotRead); } @@ -668,7 +678,7 @@ pub fn render_pad_cancellable( }) .map_err(RenderError::from)?; if let (Some(n), Some((_, end))) = (remaining, selection) { - finish_video_tail( + finish_tail( &mut writer, &probe, end as f64 / sample_rate, @@ -796,7 +806,7 @@ pub fn render_channel_cancellable( ) .map_err(RenderError::from)?; } - finish_video_tail( + finish_tail( &mut writer, &probe, source_start.max(0.0) + duration, @@ -1144,24 +1154,59 @@ mod tests { let dst = dir.join("tail.wav"); let mut writer = WavWriter::create(&dst, 48000, 2, false).unwrap(); assert_eq!( - finish_video_tail(&mut writer, &probe, 1.0, 6, 48000.0, 2), + finish_tail(&mut writer, &probe, 1.0, 6, 48000.0, 2), Err(RenderError::CannotRead) ); probe.has_video = true; assert_eq!( - finish_video_tail(&mut writer, &probe, 0.9, 6, 48000.0, 2), + finish_tail(&mut writer, &probe, 0.9, 6, 48000.0, 2), Err(RenderError::CannotRead) ); assert_eq!( - finish_video_tail(&mut writer, &probe, 1.0, 4800, 48000.0, 2), + finish_tail(&mut writer, &probe, 1.0, 4800, 48000.0, 2), Err(RenderError::CannotRead) ); - finish_video_tail(&mut writer, &probe, 1.0, 6, 48000.0, 2).unwrap(); + finish_tail(&mut writer, &probe, 1.0, 6, 48000.0, 2).unwrap(); writer.finalize().unwrap(); assert_eq!(read_pad_samples(&dst), vec![0.0; 12]); std::fs::remove_dir_all(dir).unwrap(); } + /// AC-3 in MPEG-TS decodes 256 samples late. Trimming them leaves the + /// container's last 256 samples without audio, yet a stem of the whole + /// file keeps the container's length and its audio stays on time. + #[test] + fn stems_of_audio_with_a_trimmed_decoder_delay_keep_their_length() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("recorder.mts"); + if !crate::ff::write_ac3_transport_stream(&source, 0.5, 1.0) { + return; + } + let backend = crate::portable::PortableBackend; + let probe = backend.inspect(&source).unwrap(); + assert!(!probe.has_video); + let stem = dir.path().join("stem.wav"); + render_channel_cancellable( + &backend, + &source, + &stem, + 0, + None, + false, + 0.0, + probe.duration_seconds, + 0, + &NEVER_CANCELLED, + ) + .unwrap(); + let samples = read_pad_samples(&stem); + assert_eq!( + samples.len(), + (probe.duration_seconds * 48_000.0).round() as usize + ); + assert!(crate::ff::click_at(&samples).abs_diff(24_000) <= 2); + } + #[test] fn placement_pad_invalid_clock_preserves_existing_destination() { let dir = tmp("pad-clock-underflow");