Trim the AC-3 decoder delay in transport streams

crates/align-decode/src/apple.rs

@@ -503,6 +503,7 @@ impl MediaBackend for AppleNativeBackend {
                 channels,
                 bit_depth,
                 is_float,
+                decoder_delay: 0.0,
             });
         }
         let video = video_probe(&asset)?;

crates/align-decode/src/backend.rs

@@ -76,6 +76,11 @@ pub struct AudioStreamProbe {
     pub channels: usize,
     pub bit_depth: Option<u32>,
     pub is_float: Option<bool>,
+    /// Seconds of decoder output before the stream's audio starts, which the
+    /// FFmpeg tier trims. Nonzero only for AC-3 and E-AC-3 in MPEG-TS
+    /// (AVCHD): their 256-sample codec delay is recorded nowhere, and
+    /// AVFoundation trims it, so both engines place the audio alike.
+    pub decoder_delay: f64,
 }
 
 #[derive(Clone, Debug, PartialEq)]

crates/align-decode/src/export.rs

@@ -1031,6 +1031,13 @@ fn export_media_file(
     }
     let ffmpeg = crate::ff::ffmpeg_bin()
         .ok_or_else(|| ExportError::Media("ffmpeg not found in PATH.".into()))?;
+    // FFmpeg decodes the recorder here, so trim the decoder delay its output
+    // keeps (AC-3 in MPEG-TS), as the backends do.
+    let decoder_delay = crate::ff::inspect(&plan.recorder_url)
+        .map_err(|error| ExportError::Media(format!("{}: {error}", plan.recorder_url.display())))?
+        .audio_streams
+        .first()
+        .map_or(0.0, |stream| stream.decoder_delay);
     let delay_samples = (plan.leading_silence.max(0.0) * 48_000.0).round() as u64;
     let filter = format!(
         "[1:a:0]atrim=duration={:.9},asetpts=PTS-STARTPTS,aresample=48000,adelay={delay_samples}S:all=1,apad=whole_dur={:.9},atrim=duration={:.9}[clean]",
@@ -1039,7 +1046,11 @@ fn export_media_file(
     let mut child = crate::ff::command(ffmpeg)
         .args(["-y", "-v", "error", "-i"])
         .arg(&plan.video_url)
-        .args(["-ss", &format!("{:.9}", plan.recorder_source_in), "-i"])
+        .args([
+            "-ss",
+            &format!("{:.9}", plan.recorder_source_in + decoder_delay),
+            "-i",
+        ])
         .arg(&plan.recorder_url)
         .args(["-filter_complex", &filter])
         .args(["-map", "0:v:0", "-map", "[clean]"])
@@ -1677,4 +1688,56 @@ mod tests {
         assert!((24_000..24_100).contains(&first_signal));
         let _ = std::fs::remove_dir_all(&dir);
     }
+
+    /// FFmpeg remuxes cameras outside QuickTime and MP4 and then decodes the
+    /// recorder itself. It must trim the recorder's AC-3 decoder delay as the
+    /// backends do, or the clean audio lands 256 samples late.
+    #[test]
+    fn ffmpeg_remux_trims_the_recorder_decoder_delay() {
+        let Some(ffmpeg) = crate::ff::ffmpeg_bin() else {
+            eprintln!("SKIP: ffmpeg is unavailable");
+            return;
+        };
+        let dir = tempfile::tempdir().unwrap();
+        let video = dir.path().join("camera.mts");
+        let recorder = dir.path().join("recorder.mts");
+        let video_ok = crate::ff::command(&ffmpeg)
+            .args(["-y", "-v", "error", "-f", "lavfi", "-i"])
+            .arg("color=c=black:s=160x90:r=25:d=2")
+            .args(["-c:v", "mpeg2video", "-f", "mpegts"])
+            .arg(&video)
+            .status()
+            .is_ok_and(|status| status.success());
+        if !video_ok || !crate::ff::write_ac3_transport_stream(&recorder, 0.25, 1.0) {
+            eprintln!("SKIP: fixture generation is unsupported by this ffmpeg");
+            return;
+        }
+        let plan = MediaFilePlan {
+            video_url: video,
+            recorder_url: recorder,
+            recorder_source_in: 0.1,
+            leading_silence: 0.5,
+            recorder_duration: 0.9,
+            video_duration: 2.0,
+            url: dir.path().join("out.mov"),
+        };
+
+        export_media_file(
+            &crate::portable::PortableBackend,
+            &plan,
+            &std::sync::atomic::AtomicBool::new(false),
+        )
+        .unwrap();
+
+        let mut left = Vec::new();
+        crate::portable::PortableBackend
+            .decode_native(&plan.url, 0, None, &mut |block| {
+                left.extend_from_slice(&block.frames[0]);
+                Ok(())
+            })
+            .unwrap();
+        // 0.5 s of leading silence, then the click 0.15 s into the recorder.
+        let at = crate::ff::click_at(&left);
+        assert!(at.abs_diff(31_200) <= 2, "click at {at}");
+    }
 }

crates/align-decode/src/ff.rs

@@ -115,6 +115,13 @@ fn num(value: &serde_json::Value) -> Option<f64> {
 pub fn parse_ffprobe_json(json: &[u8]) -> Option<ProbeReport> {
     let v: serde_json::Value = serde_json::from_slice(json).ok()?;
     let streams = v.get("streams")?.as_array()?;
+    // MPEG-TS records no edit list, so FFmpeg's output keeps AC-3's codec
+    // delay; in MP4 and MOV FFmpeg trims it through the edit list.
+    let transport_stream = v
+        .get("format")
+        .and_then(|f| f.get("format_name"))
+        .and_then(|n| n.as_str())
+        .is_some_and(|n| n.split(',').any(|n| n == "mpegts"));
     let mut audio_streams = Vec::new();
     let mut has_video = false;
     for s in streams {
@@ -128,14 +135,21 @@ pub fn parse_ffprobe_json(json: &[u8]) -> Option<ProbeReport> {
                 } else {
                     None
                 };
+                let sample_rate = num(s.get("sample_rate")?)?;
+                let decoder_delay = if transport_stream && matches!(codec, "ac3" | "eac3") {
+                    256.0 / sample_rate
+                } else {
+                    0.0
+                };
                 audio_streams.push(AudioStreamProbe {
-                    sample_rate: num(s.get("sample_rate")?)?,
+                    sample_rate,
                     channels: s.get("channels")?.as_u64()? as usize,
                     bit_depth: s
                         .get("bits_per_sample")
                         .and_then(|b| b.as_u64())
                         .map(|b| b as u32),
                     is_float,
+                    decoder_delay,
                 })
             }
             "video" => {
@@ -612,6 +626,9 @@ impl Drop for AudioPipeI32 {
     }
 }
 
+/// A window duration that reaches the end of any file.
+pub(crate) const WHOLE_FILE: f64 = 1e9;
+
 /// Full-file fallback decode to 8 kHz mono (fingerprint path).
 pub fn decode_mono_8k(
     path: &Path,
@@ -623,15 +640,15 @@ pub fn decode_mono_8k(
 ) -> Result<(), DecodeError> {
     // Native rate unknown here — inspect first (header-only, cheap).
     let probe = inspect(path)?;
-    let sample_rate = probe
+    let (sample_rate, delay) = probe
         .audio_streams
         .get(stream_index)
-        .map(|s| s.sample_rate)
-        .unwrap_or(0.0);
+        .map_or((0.0, 0.0), |s| (s.sample_rate, s.decoder_delay));
     if sample_rate <= 0.0 {
         return Err(DecodeError::InvalidPcm);
     }
-    let mut pipe = AudioPipe::spawn(path, stream_index, None, channels)?;
+    let window = (delay > 0.0).then_some((delay, WHOLE_FILE));
+    let mut pipe = AudioPipe::spawn(path, stream_index, window, channels)?;
     let mut mono = crate::mono::MonoPipe::new(
         sample_rate,
         8000.0,
@@ -660,17 +677,21 @@ pub fn decode_window(
     mix_channels: bool,
 ) -> Result<(f64, Vec<f32>), DecodeError> {
     let probe = inspect(path)?;
-    let sample_rate = probe
+    let (sample_rate, delay) = probe
         .audio_streams
         .get(stream_index)
-        .map(|s| s.sample_rate)
-        .unwrap_or(0.0);
+        .map_or((0.0, 0.0), |s| (s.sample_rate, s.decoder_delay));
     if sample_rate <= 0.0 {
         return Err(DecodeError::InvalidPcm);
     }
     let actual_start = start.max(0.0);
     let mut samples = Vec::new();
-    let mut pipe = AudioPipe::spawn(path, stream_index, Some((actual_start, duration)), channels)?;
+    let mut pipe = AudioPipe::spawn(
+        path,
+        stream_index,
+        Some((actual_start + delay, duration)),
+        channels,
+    )?;
     let mut mono = crate::mono::MonoPipe::new(
         sample_rate,
         16_000.0,
@@ -692,8 +713,67 @@ pub fn decode_window(
     Ok((actual_start, samples))
 }
 
+/// Write stereo 48 kHz AC-3 in MPEG-TS, silent except for a 2 ms click at
+/// `click` seconds. False (the test skips) without an AC-3 encoder.
+#[cfg(test)]
+pub(crate) fn write_ac3_transport_stream(path: &Path, click: f64, seconds: f64) -> bool {
+    let Some(ffmpeg) = ffmpeg_bin() else {
+        return false;
+    };
+    let source = format!(
+        "aevalsrc='if(between(t,{click},{}),0.9,0)':s=48000:d={seconds}",
+        click + 0.002
+    );
+    let ok = command(&ffmpeg)
+        .args(["-v", "error", "-y", "-f", "lavfi", "-i", &source])
+        .args(["-ac", "2", "-c:a", "ac3", "-b:a", "448k", "-f", "mpegts"])
+        .arg(path)
+        .status()
+        .is_ok_and(|s| s.success());
+    if !ok {
+        eprintln!("SKIP: no AC-3 encoder");
+    }
+    ok
+}
+
+/// Index of the first sample of a `write_ac3_transport_stream` click.
+#[cfg(test)]
+pub(crate) fn click_at(samples: &[f32]) -> usize {
+    samples.iter().position(|s| s.abs() > 0.3).unwrap()
+}
+
 #[cfg(test)]
 mod tests {
+    /// AC-3 decodes 256 samples late, and MPEG-TS records it nowhere (MP4
+    /// and MOV use an edit list, which FFmpeg applies). AVCHD audio must sit
+    /// where AVFoundation places it: on its timeline, not 5.3 ms early.
+    #[test]
+    fn ac3_in_transport_streams_starts_on_time() {
+        use crate::backend::MediaBackend;
+        let dir = tempfile::tempdir().unwrap();
+        let clip = dir.path().join("00001.MTS");
+        if !write_ac3_transport_stream(&clip, 1.5, 4.0) {
+            return;
+        }
+        let backend = crate::portable::PortableBackend;
+        for start in [0.0, 0.7] {
+            let mut left = Vec::new();
+            let actual = backend
+                .decode_native(&clip, 0, Some((start, None)), &mut |block| {
+                    left.extend_from_slice(&block.frames[0]);
+                    Ok(())
+                })
+                .unwrap();
+            let at = click_at(&left) + (actual * 48_000.0).round() as usize;
+            assert!(at.abs_diff(72_000) <= 2, "from {start} s: click at {at}");
+        }
+        let (window_start, window) = backend
+            .decode_window_16k(&clip, 1.0, 1.0, align_core::AudioAnalysisSource::default())
+            .unwrap();
+        let at = window_start * 1000.0 + click_at(&window) as f64 / 16.0;
+        assert!((at - 1500.0).abs() < 0.3, "window click at {at} ms");
+    }
+
     #[test]
     fn pipe_fragments_preserve_complete_multichannel_frames() {
         for channels in [1, 2, 3, 6, 8] {

crates/align-decode/src/isobmff.rs

@@ -327,6 +327,7 @@ impl Movie {
                 channels: a.channels,
                 bit_depth: a.bit_depth,
                 is_float: a.is_float,
+                decoder_delay: 0.0,
             })
             .collect();
         let video = self.video().map(|track| VideoProbe {

crates/align-decode/src/pcm.rs

@@ -193,6 +193,7 @@ impl PcmTrack {
                 channels: self.channels,
                 bit_depth: Some(self.bits),
                 is_float: Some(self.float),
+                decoder_delay: 0.0,
             }],
             has_video: false,
             video: None,

crates/align-decode/src/portable.rs

@@ -71,7 +71,10 @@ impl MediaBackend for PortableBackend {
         let mut pipe = crate::ff::AudioPipe::spawn(
             path,
             stream_index,
-            Some((actual, duration.unwrap_or(1e9))),
+            Some((
+                actual + stream.decoder_delay,
+                duration.unwrap_or(crate::ff::WHOLE_FILE),
+            )),
             channels,
         )?;
         let ok = pipe.pump(&mut |frames| {
@@ -122,7 +125,10 @@ impl MediaBackend for PortableBackend {
         let mut pipe = crate::ff::AudioPipeI32::spawn(
             path,
             stream_index,
-            Some((actual, duration.unwrap_or(1e9))),
+            Some((
+                actual + stream.decoder_delay,
+                duration.unwrap_or(crate::ff::WHOLE_FILE),
+            )),
             channels,
         )?;
         let ok = pipe.pump(&mut |frames| {