""" Continuous PulseAudio ring buffer + per-call slicing. The ring-buffer design is deliberate and load-bearing: a persistent capture process runs for the lifetime of the node and every call is cut out of the buffer after the fact. Spawning FFmpeg per call used to lose the first 1-2 s to process startup, which meant short transmissions produced empty files. Nothing about that changes here — only the INPUT changes, from an HTTP GET on Icecast to a PulseAudio monitor capture. Why not Icecast: it lags ~1 s at connect and drifts progressively to 100 s+, so slice timestamps and audio content diverge without bound. Icecast stays in the stack for frontend/mobile live listening; it is not an accuracy path. CLOCK DOMAIN: chunks are stamped with time.time(), the host wall clock. OP25's call_log timestamps are time.time() from inside the op25 container. All client containers run network_mode: host and share the host kernel clock, so the two are the same clock and the pre-roll arithmetic below is a direct subtraction with no offset mapping. (A wall-clock STEP — e.g. a large NTP correction — would corrupt at most the calls in flight at that instant; the buffer self-heals within RING_BUFFER_SECONDS.) """ import asyncio import time from collections import deque from datetime import datetime, timezone from pathlib import Path from typing import List, Optional, Tuple import httpx from app.config import settings from app.internal import credentials, pulse from app.internal.logger import logger # Safety cap on a single recording; mirrors MAX_SEGMENT_SECONDS in metadata_watcher. MAX_RECORDING_SECONDS = 600 # Audio included ahead of OP25's call_log timestamp. The grant is logged when the # channel is granted, so the first syllable can land marginally before it. PRE_ROLL_SECONDS = 0.25 # Rolling history kept when no call is active. Budget for the worst realistic # detection latency: 0.5 s poll interval + ~0.2 s http_server blocking floor + # up to 3 s httpx timeout on a stalled poll + callback work ≈ 4 s from grant to # start_recording(). 30 s is ~7x that margin, and at 16 kbps costs only ~60 KB of # RAM, so there is no reason to trim it closer. The buffer is what makes # detection latency harmless: however late we notice, we seek back to OP25's # exact timestamp. RING_BUFFER_SECONDS = 30 # ~128 ms of audio per chunk at 16 kbps. Chunk size IS the timestamp resolution of # the ring buffer, so it must stay well under PRE_ROLL_SECONDS — the old 4096-byte # reads were ~2 s per chunk, which made sub-second slicing meaningless. READ_CHUNK_BYTES = 256 # Encoder settings, matched on purpose to what Liquidsoap already pushes to # Icecast — %mp3(bitrate=16, samplerate=22050, stereo=false) — so the C2 /upload # endpoint keeps receiving exactly the kind of MP3 it has always received # (multipart "audio/mpeg", stored to GCS as .mp3, then fed to Whisper). # Change both of these together if you ever want higher-fidelity uploads. MP3_BITRATE = "16k" MP3_SAMPLE_RATE = "22050" # Backoff bounds for restarting a dead capture process. RESTART_BACKOFF_MIN = 1.0 RESTART_BACKOFF_MAX = 15.0 class CallRecorder: """Continuous PulseAudio capture into a ring buffer, sliced per call.""" def __init__(self): self._recordings_dir = Path(settings.recordings_path) # Ring buffer: deque of (wall_clock_epoch_at_arrival, mp3_bytes) self._buffer: deque[Tuple[float, bytes]] = deque() self._buffer_bytes: int = 0 self._stream_task: Optional[asyncio.Task] = None self._proc: Optional[asyncio.subprocess.Process] = None self._capturing: bool = False # Active recording state self._call_id: Optional[str] = None self._call_start: Optional[float] = None # OP25 grant epoch self._slice_start: Optional[float] = None # _call_start - PRE_ROLL_SECONDS # ------------------------------------------------------------------ # Lifecycle # ------------------------------------------------------------------ async def start(self) -> None: """Start the persistent capture. Call once from app lifespan.""" self._stream_task = asyncio.create_task(self._capture_loop()) logger.info("PulseAudio ring-buffer starting.") async def stop(self) -> None: if self._stream_task: self._stream_task.cancel() try: await self._stream_task except asyncio.CancelledError: pass self._stream_task = None await self._terminate_proc() # ------------------------------------------------------------------ # Capture # ------------------------------------------------------------------ def _ffmpeg_command(self) -> List[str]: return [ "ffmpeg", "-hide_banner", "-nostdin", "-nostats", "-loglevel", "warning", "-f", "pulse", "-i", settings.pulse_source, "-ac", "1", "-ar", MP3_SAMPLE_RATE, "-b:a", MP3_BITRATE, # Without this, the MP3 muxer fills its 32 KB AVIO buffer before # writing anything — 16 s of audio per burst at 16 kbps, which would # destroy the arrival timestamps the slicing depends on. "-flush_packets", "1", "-f", "mp3", "-", ] async def _capture_loop(self) -> None: backoff = RESTART_BACKOFF_MIN while True: try: # The original bug this path was abandoned for: FFmpeg was launched # with -f pulse before the shared socket existed, failed instantly, # and never recovered. Wait for it, bounded, every time. if not await pulse.wait_until_ready(): await asyncio.sleep(backoff) backoff = min(backoff * 2, RESTART_BACKOFF_MAX) continue await self._run_capture() logger.warning("PulseAudio capture process exited — restarting.") except asyncio.CancelledError: await self._terminate_proc() raise except Exception as e: logger.warning(f"PulseAudio capture error ({e}) — restarting.") self._capturing = False await asyncio.sleep(backoff) backoff = min(backoff * 2, RESTART_BACKOFF_MAX) async def _run_capture(self) -> None: cmd = self._ffmpeg_command() logger.info(f"Starting capture: ffmpeg -f pulse -i {settings.pulse_source}") proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) self._proc = proc stderr_task = asyncio.create_task(self._drain_stderr(proc)) try: assert proc.stdout is not None while True: chunk = await proc.stdout.read(READ_CHUNK_BYTES) if not chunk: break # EOF — FFmpeg died or the source went away if not self._capturing: self._capturing = True logger.info("PulseAudio capture is producing audio.") self._ingest(chunk) finally: self._capturing = False # _drain_stderr swallows its own CancelledError, so it finishes cleanly # and never needs awaiting here. stderr_task.cancel() await self._terminate_proc() async def _drain_stderr(self, proc: asyncio.subprocess.Process) -> None: """Surface FFmpeg's diagnostics instead of letting the pipe fill and block.""" if proc.stderr is None: return try: while True: line = await proc.stderr.readline() if not line: return logger.warning(f"ffmpeg(pulse): {line.decode(errors='replace').strip()}") except asyncio.CancelledError: return except Exception: return async def _terminate_proc(self) -> None: proc, self._proc = self._proc, None if proc is None or proc.returncode is not None: return try: # Synchronous, so the signal lands even if we are being cancelled and # the reap below never gets to run. proc.terminate() except Exception: return try: await asyncio.wait_for(proc.wait(), timeout=5) except asyncio.TimeoutError: try: proc.kill() except Exception: pass except asyncio.CancelledError: raise except Exception: pass def _ingest(self, chunk: bytes) -> None: """Append a chunk and trim stale audio off the front of the buffer.""" now = time.time() self._buffer.append((now, chunk)) self._buffer_bytes += len(chunk) # While recording, never trim anything the current slice still needs — but # keep a hard ceiling so a recording that somehow never closes can't grow # the buffer without bound. if self._slice_start is not None: keep_from = max( self._slice_start, now - (MAX_RECORDING_SECONDS + RING_BUFFER_SECONDS), ) else: keep_from = now - RING_BUFFER_SECONDS while self._buffer and self._buffer[0][0] < keep_from: _, old = self._buffer.popleft() self._buffer_bytes -= len(old) # ------------------------------------------------------------------ # Recording API # ------------------------------------------------------------------ async def start_recording(self, call_id: str, start_epoch: Optional[float] = None) -> bool: """ Open a recording. `start_epoch` is OP25's call_log timestamp (host wall clock); the slice begins PRE_ROLL_SECONDS before it. Omit it only when no OP25 timestamp is available — then we fall back to "now", losing precision. """ if self._call_id: logger.warning(f"Recording already active ({self._call_id}) — ignoring start for {call_id}.") return False self._call_id = call_id self._call_start = start_epoch if start_epoch else time.time() self._slice_start = self._call_start - PRE_ROLL_SECONDS if not self._capturing: logger.warning(f"Recording {call_id} opened while PulseAudio capture is down — audio may be missing.") oldest = self._buffer[0][0] if self._buffer else None if oldest is not None and self._slice_start < oldest: # Pre-roll predates the buffer: node just started, capture restarted, # or OP25's timestamp is far in the past. Clamp and say so. logger.warning( f"Pre-roll for call {call_id} predates buffered audio by " f"{oldest - self._slice_start:.2f}s — recording starts at the buffer head." ) logger.info(f"Recording started: {call_id} (slice from {self._slice_start:.3f})") return True async def stop_recording(self, end_epoch: Optional[float] = None) -> Optional[Path]: """Close the recording and write the slice. `end_epoch` is host wall clock.""" if not self._call_id: return None call_id = self._call_id call_start = self._call_start slice_start = self._slice_start self._call_id = None self._call_start = None self._slice_start = None end = end_epoch if end_epoch else time.time() if call_start is not None: end = min(end, call_start + MAX_RECORDING_SECONDS) if slice_start is None: return None chunks: List[bytes] = [] for ts, chunk in self._buffer: if ts < slice_start: continue chunks.append(chunk) if ts >= end: # Include the chunk straddling `end` so the tail is never clipped, # then stop. break if not chunks: logger.warning( f"No buffered audio for call {call_id} " f"(window {slice_start:.3f}–{end:.3f}) — PulseAudio capture may be down." ) return None self._recordings_dir.mkdir(parents=True, exist_ok=True) ts_str = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") output_path = self._recordings_dir / f"{ts_str}_{call_id}.mp3" output_path.write_bytes(b"".join(chunks)) size = output_path.stat().st_size if size > 0: logger.info(f"Recording saved: {output_path.name} ({size} bytes, {end - slice_start:.2f}s window)") return output_path output_path.unlink(missing_ok=True) logger.warning(f"Recording for call {call_id} produced an empty file.") return None # ------------------------------------------------------------------ # Upload (unchanged interface) # ------------------------------------------------------------------ async def upload_recording( self, file_path: Path, call_id: str, talkgroup_id: Optional[int] = None, talkgroup_name: Optional[str] = None, system_id: Optional[str] = None, ) -> Optional[str]: if not settings.c2_url: logger.info("No C2_URL configured — skipping upload.") return None upload_url = f"{settings.c2_url}/upload" api_key = credentials.get_api_key() headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} form: dict = {"call_id": call_id, "node_id": settings.node_id} if talkgroup_id is not None: form["talkgroup_id"] = str(talkgroup_id) if talkgroup_name: form["talkgroup_name"] = talkgroup_name if system_id: form["system_id"] = system_id try: async with httpx.AsyncClient(timeout=120) as client: with open(file_path, "rb") as f: r = await client.post( upload_url, files={"file": (file_path.name, f, "audio/mpeg")}, data=form, headers=headers, ) r.raise_for_status() audio_url = r.json().get("url") logger.info(f"Upload complete: {audio_url}") return audio_url except Exception as e: logger.error(f"Upload failed: {e}") return None finally: try: file_path.unlink() except Exception: pass # ------------------------------------------------------------------ # State # ------------------------------------------------------------------ @property def is_recording(self) -> bool: return self._call_id is not None @property def is_capturing(self) -> bool: """True when FFmpeg is alive and audio is actually arriving.""" return self._capturing @property def buffered_seconds(self) -> float: if len(self._buffer) < 2: return 0.0 return self._buffer[-1][0] - self._buffer[0][0] call_recorder = CallRecorder()