157c783e25
Fills the foundation stubs in ingest.py and audio_sync.py. - ingest: ffprobe probe + mono 16 kHz WAV extraction per video (ffmpeg); idempotent run_ingest reuses existing rows so synthetic -> ingest -> sync works without unique-constraint collisions. - sync: PHAT-whitened GCC-PHAT pairwise offsets with sub-sample parabolic peak refinement, scored by peak-to-second-peak ratio; confidence-weighted global least-squares solve with cycle-consistency rejection (>50 ms); windowed drift estimation (ppm) with a 5 ppm deadband; disconnected sync-graph components -> offset_ms=None (spec pitfall #5). - Persists offset_ms/drift_ppm/sync_confidence via db.update_video_sync and exports data/work/sync.json. Verified on the synthetic fixture: synthetic -> ingest -> sync recovers the ground-truth offsets (0 / +1370 / -842 ms) to <0.001 ms and drift 0 ppm, well inside the spec M1 tolerances (+/-10 ms, +/-3 ppm). pytest: 43 passed (24 foundation + 19 new in test_ingest.py and test_audio_sync.py). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
241 lines
10 KiB
Python
241 lines
10 KiB
Python
"""Audio-sync tests (spec M1, lane A).
|
||
|
||
Pure-numpy where possible (no ffmpeg): GCC-PHAT sign/precision, pairwise+global solve
|
||
recovering the M0 known offsets (±10 ms), disconnected components → None (pitfall #5),
|
||
cycle-consistency rejection, drift recovery (±3 ppm), and the full ``run_sync`` persist +
|
||
export path against synthesized WAVs. One ffmpeg-gated test runs the literal spec M1
|
||
acceptance: ``synthetic -> ingest -> sync`` recovers ground truth.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import shutil
|
||
|
||
import numpy as np
|
||
import pytest
|
||
|
||
from festival4d import audio_sync, config, db, ingest, synthetic
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# helpers
|
||
# ---------------------------------------------------------------------------
|
||
def _broadband(n: int, seed: int = 0) -> np.ndarray:
|
||
"""Seeded, mildly band-limited noise — a sharp, unambiguous correlation target."""
|
||
rng = np.random.default_rng(seed)
|
||
x = rng.standard_normal(n)
|
||
return np.convolve(x, np.ones(4) / 4.0, mode="same")
|
||
|
||
|
||
def _delay_samples(x: np.ndarray, d: int) -> np.ndarray:
|
||
"""Return ``y`` with ``y[n] = x[n - d]`` (x delayed by d samples, zero-filled)."""
|
||
y = np.zeros_like(x)
|
||
if d >= 0:
|
||
y[d:] = x[:len(x) - d] if d else x
|
||
else:
|
||
y[:d] = x[-d:]
|
||
return y
|
||
|
||
|
||
def _clean_audio_dir() -> None:
|
||
"""Remove any WAVs left in the shared session audio dir (ids are reused after reset_db)."""
|
||
config.AUDIO_DIR.mkdir(parents=True, exist_ok=True)
|
||
for wav in config.AUDIO_DIR.glob("*.wav"):
|
||
wav.unlink()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# GCC-PHAT
|
||
# ---------------------------------------------------------------------------
|
||
def test_gcc_phat_recovers_known_delay_positive_and_negative():
|
||
sr = 16_000
|
||
ref = _broadband(sr * 4)
|
||
for d in (0, 137, -211, 800):
|
||
sig = _delay_samples(ref, d)
|
||
offset_s, conf = audio_sync.gcc_phat(sig, ref, sr)
|
||
# positive offset_s means sig lags ref, i.e. matches a positive sample delay
|
||
assert abs(offset_s - d / sr) < 0.5 / sr, (d, offset_s * sr)
|
||
assert conf > 5.0
|
||
|
||
|
||
def test_gcc_phat_confidence_drops_with_noise():
|
||
sr = 16_000
|
||
ref = _broadband(sr * 4, seed=1)
|
||
clean = _delay_samples(ref, 300)
|
||
rng = np.random.default_rng(2)
|
||
noisy = clean + 3.0 * rng.standard_normal(len(clean))
|
||
_, c_clean = audio_sync.gcc_phat(clean, ref, sr)
|
||
_, c_noisy = audio_sync.gcc_phat(noisy, ref, sr)
|
||
assert c_clean > c_noisy
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# pairwise + global solve
|
||
# ---------------------------------------------------------------------------
|
||
def test_pairwise_and_solve_recover_synthetic_offsets():
|
||
"""The M0 known offsets (0/+1370/−842 ms) recovered within the spec's ±10 ms."""
|
||
sr = config.AUDIO_SAMPLE_RATE
|
||
master, _ = synthetic.synth_master_audio(sr)
|
||
signals = {
|
||
i + 1: synthetic.slice_for_offset(master, sr, off, config.SYNTH_DURATION_S)
|
||
for i, off in enumerate(config.SYNTH_OFFSETS_MS)
|
||
}
|
||
edges = audio_sync.pairwise_offsets(signals, sr)
|
||
solved = audio_sync.solve_global_offsets(edges, list(signals))
|
||
for i, off in enumerate(config.SYNTH_OFFSETS_MS):
|
||
assert solved[i + 1] is not None
|
||
assert abs(solved[i + 1] - off) <= 10.0, (off, solved[i + 1])
|
||
|
||
|
||
def test_pairwise_edge_sign_is_relative_start_offset():
|
||
"""Edge ``offset_s`` = offset_a − offset_b (seconds)."""
|
||
sr = config.AUDIO_SAMPLE_RATE
|
||
master, _ = synthetic.synth_master_audio(sr)
|
||
a = synthetic.slice_for_offset(master, sr, 500.0, 6.0) # id 1, offset +500 ms
|
||
b = synthetic.slice_for_offset(master, sr, -300.0, 6.0) # id 2, offset −300 ms
|
||
edges = audio_sync.pairwise_offsets({1: a, 2: b}, sr)
|
||
assert len(edges) == 1
|
||
assert abs(edges[0]["offset_s"] * 1000.0 - (500.0 - -300.0)) <= 10.0
|
||
|
||
|
||
def test_solve_disconnected_component_gets_none():
|
||
"""A video with no usable edges is unsynced (offset None), not guessed (pitfall #5)."""
|
||
edges = [
|
||
{"a": 1, "b": 2, "offset_s": -0.5, "confidence": 30.0},
|
||
{"a": 2, "b": 3, "offset_s": -0.4, "confidence": 30.0},
|
||
]
|
||
solved = audio_sync.solve_global_offsets(edges, [1, 2, 3, 4])
|
||
assert solved[1] == 0.0
|
||
assert abs(solved[2] - 500.0) < 1.0
|
||
assert abs(solved[3] - 900.0) < 1.0
|
||
assert solved[4] is None # isolated → None
|
||
|
||
|
||
def test_solve_rejects_cycle_inconsistent_edge():
|
||
"""A grossly inconsistent edge (>50 ms residual) is rejected; the rest still solve."""
|
||
# True offsets: 1=0, 2=+500 ms, 3=+900 ms. Edge 1-3 is corrupted (should be −0.9 s).
|
||
edges = [
|
||
{"a": 1, "b": 2, "offset_s": -0.5, "confidence": 20.0},
|
||
{"a": 2, "b": 3, "offset_s": -0.4, "confidence": 20.0},
|
||
{"a": 1, "b": 3, "offset_s": +5.0, "confidence": 20.0}, # bad
|
||
]
|
||
solved = audio_sync.solve_global_offsets(edges, [1, 2, 3])
|
||
assert solved[1] == 0.0
|
||
assert abs(solved[2] - 500.0) < 1.0
|
||
assert abs(solved[3] - 900.0) < 1.0 # recovered despite the bad edge
|
||
|
||
|
||
def test_low_confidence_edges_are_dropped():
|
||
"""Edges below the confidence floor don't connect a component."""
|
||
edges = [{"a": 1, "b": 2, "offset_s": -0.5, "confidence": 1.01}]
|
||
solved = audio_sync.solve_global_offsets(edges, [1, 2])
|
||
# 1 is the lone reference at 0; 2 has only a rejected edge → None
|
||
assert solved[1] == 0.0 and solved[2] is None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# drift
|
||
# ---------------------------------------------------------------------------
|
||
def _make_drifted(ref: np.ndarray, sr: int, drift_ppm: float, offset_s: float = 0.0) -> np.ndarray:
|
||
"""Build v from the config timebase: v(τ) = ref((1 − d)·τ + offset_s), d = ppm·1e−6."""
|
||
d = drift_ppm * 1e-6
|
||
local = np.arange(len(ref))
|
||
src = (1.0 - d) * local + offset_s * sr
|
||
return np.interp(src, np.arange(len(ref)), ref, left=0.0, right=0.0)
|
||
|
||
|
||
def test_estimate_drift_recovers_known_ppm():
|
||
sr = 8_000
|
||
ref = _broadband(sr * 150, seed=5) # 150 s so several 30 s-spaced windows fit
|
||
for ppm in (30.0, -22.0):
|
||
v = _make_drifted(ref, sr, ppm)
|
||
est = audio_sync.estimate_drift(v, ref, sr)
|
||
assert abs(est - ppm) <= 3.0, (ppm, est)
|
||
|
||
|
||
def test_estimate_drift_zero_within_deadband():
|
||
sr = 8_000
|
||
ref = _broadband(sr * 150, seed=6)
|
||
assert audio_sync.estimate_drift(ref, ref, sr) == 0.0 # identical
|
||
assert audio_sync.estimate_drift(_make_drifted(ref, sr, 1.5), ref, sr) == 0.0 # < 5 ppm
|
||
|
||
|
||
def test_estimate_drift_returns_zero_when_too_short():
|
||
sr = 16_000
|
||
ref = _broadband(sr * 15) # < 2 windows at 10 s / 30 s hop
|
||
assert audio_sync.estimate_drift(ref, ref, sr) == 0.0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# run_sync orchestration
|
||
# ---------------------------------------------------------------------------
|
||
def test_run_sync_persists_and_exports():
|
||
"""Full run_sync against synthesized WAVs (no ffmpeg): DB + sync.json, ±10 ms."""
|
||
sr = config.AUDIO_SAMPLE_RATE
|
||
master, _ = synthetic.synth_master_audio(sr)
|
||
db.init_engine()
|
||
db.reset_db()
|
||
_clean_audio_dir()
|
||
gt = {}
|
||
for i, off in enumerate(config.SYNTH_OFFSETS_MS):
|
||
v = db.add_video(filename=f"cam{i}.mp4", duration_s=config.SYNTH_DURATION_S,
|
||
fps=30.0, width=640, height=360)
|
||
clip = synthetic.slice_for_offset(master, sr, off, config.SYNTH_DURATION_S)
|
||
synthetic.write_wav(ingest.audio_wav_path(v.id), clip, sr)
|
||
gt[v.id] = off
|
||
|
||
solution = audio_sync.run_sync()
|
||
|
||
for v in db.get_videos():
|
||
assert v.offset_ms is not None
|
||
assert abs(v.offset_ms - gt[v.id]) <= 10.0, (gt[v.id], v.offset_ms)
|
||
assert v.drift_ppm == 0.0
|
||
assert 0.0 <= v.sync_confidence <= 1.0
|
||
ref = min(gt)
|
||
assert solution["reference_video_id"] == ref
|
||
assert db.get_video(ref).sync_confidence == 1.0
|
||
exported = json.loads(config.SYNC_JSON.read_text())
|
||
assert exported["reference_video_id"] == ref
|
||
assert len(exported["edges"]) == 3
|
||
|
||
|
||
def test_run_sync_marks_video_without_audio_unsynced():
|
||
"""A registered video with no extracted WAV is left offset=None, not guessed."""
|
||
sr = config.AUDIO_SAMPLE_RATE
|
||
master, _ = synthetic.synth_master_audio(sr)
|
||
db.init_engine()
|
||
db.reset_db()
|
||
_clean_audio_dir()
|
||
v0 = db.add_video(filename="cam0.mp4", duration_s=20.0, fps=30.0, width=640, height=360)
|
||
v1 = db.add_video(filename="cam1.mp4", duration_s=20.0, fps=30.0, width=640, height=360)
|
||
no_audio = db.add_video(filename="cam2.mp4", duration_s=20.0, fps=30.0, width=640, height=360)
|
||
synthetic.write_wav(ingest.audio_wav_path(v0.id),
|
||
synthetic.slice_for_offset(master, sr, 0.0, 20.0), sr)
|
||
synthetic.write_wav(ingest.audio_wav_path(v1.id),
|
||
synthetic.slice_for_offset(master, sr, 1370.0, 20.0), sr)
|
||
# no WAV for `no_audio`
|
||
|
||
audio_sync.run_sync()
|
||
assert db.get_video(no_audio.id).offset_ms is None
|
||
assert db.get_video(no_audio.id).sync_confidence is None
|
||
assert db.get_video(v0.id).offset_ms == 0.0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ffmpeg-gated: the literal spec M1 acceptance (synthetic -> ingest -> sync)
|
||
# ---------------------------------------------------------------------------
|
||
@pytest.mark.skipif(shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None,
|
||
reason="ffmpeg/ffprobe required for the end-to-end sync acceptance")
|
||
def test_synthetic_ingest_sync_recovers_ground_truth():
|
||
synthetic.build(duration_s=8.0, run_ffmpeg=True) # into the temp data dir (conftest)
|
||
ingest.run_ingest()
|
||
audio_sync.run_sync()
|
||
|
||
gt = {v["filename"]: v["offset_ms"]
|
||
for v in json.loads(config.GROUND_TRUTH_JSON.read_text())["videos"]}
|
||
for v in db.get_videos():
|
||
assert v.offset_ms is not None, v.filename
|
||
assert abs(v.offset_ms - gt[v.filename]) <= 10.0, (v.filename, v.offset_ms, gt[v.filename])
|
||
assert abs(v.drift_ppm) <= 3.0
|