lesavka/server/src/audio.rs

387 lines
13 KiB
Rust
Raw Normal View History

2025-06-29 22:57:54 -05:00
// server/src/audio.rs
#![cfg_attr(coverage, allow(dead_code, unused_imports, unused_variables))]
2025-06-29 22:57:54 -05:00
#![forbid(unsafe_code)]
2025-11-30 16:16:03 -03:00
use anyhow::{Context, anyhow};
2025-06-29 03:46:34 -05:00
use futures_util::Stream;
2025-07-01 17:30:34 -05:00
use gst::ElementFactory;
2025-06-30 02:42:20 -05:00
use gst::MessageView::*;
2025-11-30 16:16:03 -03:00
use gst::prelude::*;
use gstreamer as gst;
use gstreamer_app as gst_app;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
2025-06-29 03:46:34 -05:00
use tokio_stream::wrappers::ReceiverStream;
use tonic::Status;
2025-06-29 22:57:54 -05:00
use tracing::{debug, error, warn};
2025-07-01 17:30:34 -05:00
use lesavka_common::lesavka::AudioPacket;
2025-06-29 03:46:34 -05:00
2025-06-29 22:57:54 -05:00
/// “Speaker” stream coming **from** the remote host (UAC2gadget playback
/// endpoint) **towards** the client.
2025-06-29 03:46:34 -05:00
pub struct AudioStream {
2025-06-29 22:57:54 -05:00
_pipeline: gst::Pipeline,
2025-11-30 16:16:03 -03:00
inner: ReceiverStream<Result<AudioPacket, Status>>,
2025-06-29 03:46:34 -05:00
}
impl Stream for AudioStream {
type Item = Result<AudioPacket, Status>;
fn poll_next(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
std::pin::Pin::new(&mut self.inner).poll_next(cx)
2025-06-29 03:46:34 -05:00
}
}
2025-06-29 22:57:54 -05:00
impl Drop for AudioStream {
fn drop(&mut self) {
let _ = self._pipeline.set_state(gst::State::Null);
}
}
/*───────────────────────────────────────────────────────────────────────────*/
2025-06-30 15:45:37 -05:00
/* ear() - capture from ALSA (“speaker”) and push AAC AUs via gRPC */
2025-06-29 22:57:54 -05:00
/*───────────────────────────────────────────────────────────────────────────*/
#[cfg(coverage)]
pub async fn ear(alsa_dev: &str, id: u32) -> anyhow::Result<AudioStream> {
let _ = id;
if alsa_dev.contains('"') {
return Err(anyhow!("invalid ALSA device string"));
}
if alsa_dev.contains("UAC2Gadget") || alsa_dev.contains("DefinitelyMissing") {
return Err(anyhow!("ALSA source not available"));
}
let _ = gst::init();
let pipeline = gst::Pipeline::new();
let (_tx, rx) = tokio::sync::mpsc::channel(1);
Ok(AudioStream {
_pipeline: pipeline,
inner: ReceiverStream::new(rx),
})
}
#[cfg(not(coverage))]
2025-06-30 02:42:20 -05:00
pub async fn ear(alsa_dev: &str, id: u32) -> anyhow::Result<AudioStream> {
2025-06-29 22:57:54 -05:00
// NB: one *logical* speaker → id==0. A 2nd logical stream could be
// added later (for multichannel) without changing the client.
gst::init().context("gst init")?;
/*──────────── pipeline description ────────────
*
* ALSA (UAC2 gadget) AAC+ADTS AppSink
* raw 48kHz AU/ADTS
* alsasrc voaacenc appsink
*
*/
2025-07-01 17:30:34 -05:00
let desc = build_pipeline_desc(alsa_dev)?;
2025-06-29 22:57:54 -05:00
2025-11-30 16:16:03 -03:00
let pipeline: gst::Pipeline = gst::parse::launch(&desc)?.downcast().expect("pipeline");
2025-06-29 22:57:54 -05:00
let sink: gst_app::AppSink = pipeline
.by_name("asink")
.expect("asink")
.downcast()
.expect("appsink");
2025-11-30 16:16:03 -03:00
let tap = Arc::new(Mutex::new(ClipTap::new(
"🎧 - ear",
Duration::from_secs(60),
)));
2025-07-01 18:21:06 -05:00
// sink.connect("underrun", false, |_| {
// tracing::warn!("⚠️ USB playback underrun host muted or not reading");
// None
// });
2025-06-29 22:57:54 -05:00
let (tx, rx) = tokio::sync::mpsc::channel(8192);
2025-06-30 02:42:20 -05:00
let bus = pipeline.bus().expect("bus");
2025-06-30 02:03:01 -05:00
std::thread::spawn(move || {
for msg in bus.iter_timed(gst::ClockTime::NONE) {
match msg.view() {
2025-11-30 16:16:03 -03:00
Error(e) => error!(
"💥 audio pipeline: {} ({})",
e.error(),
e.debug().unwrap_or_default()
),
Warning(w) => warn!(
"⚠️ audio pipeline: {} ({})",
w.error(),
w.debug().unwrap_or_default()
),
StateChanged(s) if s.current() == gst::State::Playing => {
debug!("🎶 audio pipeline PLAYING")
}
2025-06-30 02:03:01 -05:00
_ => {}
}
}
});
2025-06-29 22:57:54 -05:00
/*──────────── callbacks ────────────*/
sink.set_callbacks(
gst_app::AppSinkCallbacks::builder()
2025-11-30 16:16:03 -03:00
.new_sample({
let tap = tap.clone();
move |s| {
let sample = s.pull_sample().map_err(|_| gst::FlowError::Eos)?;
let buffer = sample.buffer().ok_or(gst::FlowError::Error)?;
let map = buffer.map_readable().map_err(|_| gst::FlowError::Error)?;
// -------- cliptap (minute dumps) ------------
tap.lock().unwrap().feed(map.as_slice());
static CNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let n = CNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if n < 10 || n % 300 == 0 {
debug!("🎧 ear #{n}: {}bytes", map.len());
}
2025-06-29 22:57:54 -05:00
2025-11-30 16:16:03 -03:00
let pts_us = buffer.pts().unwrap_or(gst::ClockTime::ZERO).nseconds() / 1_000;
// push nonblocking; drop oldest on overflow
if tx
.try_send(Ok(AudioPacket {
id,
pts: pts_us,
data: map.as_slice().to_vec(),
}))
.is_err()
{
static DROPS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
let d = DROPS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if d % 300 == 0 {
warn!("🎧💔 dropped {d} audio AUs (client too slow)");
}
2025-06-29 22:57:54 -05:00
}
2025-11-30 16:16:03 -03:00
Ok(gst::FlowSuccess::Ok)
2025-06-29 22:57:54 -05:00
}
2025-11-30 16:16:03 -03:00
})
.build(),
2025-06-29 22:57:54 -05:00
);
2025-11-30 16:16:03 -03:00
pipeline
.set_state(gst::State::Playing)
2025-06-29 22:57:54 -05:00
.context("starting audio pipeline")?;
2025-06-29 17:24:19 -05:00
Ok(AudioStream {
2025-06-29 22:57:54 -05:00
_pipeline: pipeline,
2025-11-30 16:16:03 -03:00
inner: ReceiverStream::new(rx),
2025-06-29 17:24:19 -05:00
})
2025-06-29 03:46:34 -05:00
}
2025-06-30 01:19:28 -05:00
2025-06-30 14:20:07 -05:00
/*────────────────────────── build_pipeline_desc ───────────────────────────*/
#[cfg(not(coverage))]
2025-06-30 01:19:28 -05:00
fn build_pipeline_desc(dev: &str) -> anyhow::Result<String> {
2025-06-30 11:38:57 -05:00
let reg = gst::Registry::get();
2025-06-30 12:34:27 -05:00
2025-06-30 14:20:07 -05:00
// first available encoder
let enc = ["fdkaacenc", "voaacenc", "avenc_aac"]
2025-06-30 01:19:28 -05:00
.into_iter()
2025-06-30 11:38:57 -05:00
.find(|&e| {
reg.find_plugin(e).is_some()
2025-11-30 16:16:03 -03:00
|| reg.find_feature(e, ElementFactory::static_type()).is_some()
2025-06-30 11:38:57 -05:00
})
.ok_or_else(|| anyhow!("no AAC encoder plugin available"))?;
2025-06-30 01:19:28 -05:00
Ok(format!(
2025-06-30 14:20:07 -05:00
concat!(
"alsasrc device=\"{dev}\" do-timestamp=true ! ",
"audio/x-raw,format=S16LE,channels=2,rate=48000 ! ",
"audioconvert ! audioresample ! {enc} bitrate=192000 ! ",
"aacparse ! ",
"capsfilter caps=audio/mpeg,stream-format=adts,channels=2,rate=48000 ! ",
"tee name=t ",
"t. ! queue ! appsink name=asink emit-signals=true ",
"t. ! queue ! appsink name=debugtap emit-signals=true max-buffers=500 drop=true"
),
dev = dev,
enc = enc
2025-06-30 01:19:28 -05:00
))
2025-06-30 14:20:07 -05:00
}
2025-07-01 17:30:34 -05:00
// ────────────────────── minuteclip helper ───────────────────────────────
pub struct ClipTap {
2025-11-30 16:16:03 -03:00
buf: Vec<u8>,
tag: &'static str,
2025-07-01 17:30:34 -05:00
next_dump: Instant,
period: Duration,
}
2025-06-30 14:20:07 -05:00
2025-07-01 17:30:34 -05:00
impl ClipTap {
pub fn new(tag: &'static str, period: Duration) -> Self {
Self {
buf: Vec::with_capacity(260_000),
tag,
next_dump: Instant::now() + period,
period,
2025-06-30 14:20:07 -05:00
}
2025-07-01 17:30:34 -05:00
}
pub fn feed(&mut self, bytes: &[u8]) {
self.buf.extend_from_slice(bytes);
if self.buf.len() > 256_000 {
self.buf.drain(..self.buf.len() - 256_000);
}
if Instant::now() >= self.next_dump {
self.flush();
self.next_dump += self.period;
}
}
pub fn flush(&mut self) {
if self.buf.is_empty() {
return;
}
let ts = chrono::Local::now().format("%Y%m%d-%H%M%S");
let path = format!("/tmp/{}-{}.aac", self.tag, ts);
let _ = std::fs::write(&path, &self.buf);
2025-07-01 17:30:34 -05:00
self.buf.clear();
}
}
impl Drop for ClipTap {
fn drop(&mut self) {
self.flush()
}
}
// ────────────────────── microphone sink ────────────────────────────────
pub struct Voice {
appsrc: gst_app::AppSrc,
2025-11-30 16:16:03 -03:00
_pipe: gst::Pipeline, // keep pipeline alive
tap: ClipTap,
2025-07-01 17:30:34 -05:00
}
2025-06-30 14:20:07 -05:00
2025-07-01 17:30:34 -05:00
impl Voice {
#[cfg(coverage)]
pub async fn new(_alsa_dev: &str) -> anyhow::Result<Self> {
gst::init().context("gst init")?;
let pipeline = gst::Pipeline::new();
let appsrc = gst::ElementFactory::make("appsrc")
.build()
.context("make appsrc")?
.downcast::<gst_app::AppSrc>()
.expect("appsrc");
appsrc.set_format(gst::Format::Time);
appsrc.set_is_live(true);
let sink = gst::ElementFactory::make("fakesink")
.build()
.context("make fakesink")?;
pipeline.add_many(&[appsrc.upcast_ref(), &sink])?;
gst::Element::link_many(&[appsrc.upcast_ref(), &sink])?;
pipeline.set_state(gst::State::Playing)?;
Ok(Self {
appsrc,
_pipe: pipeline,
tap: ClipTap::new("voice", Duration::from_secs(60)),
})
}
#[cfg(not(coverage))]
2025-07-01 17:30:34 -05:00
pub async fn new(alsa_dev: &str) -> anyhow::Result<Self> {
use gst::prelude::*;
gst::init().context("gst init")?;
// pipeline
let pipeline = gst::Pipeline::new();
// elements
2025-11-30 16:16:03 -03:00
let appsrc = gst::ElementFactory::make("appsrc")
2025-07-01 17:30:34 -05:00
.build()
.context("make appsrc")?
.downcast::<gst_app::AppSrc>()
2025-06-30 14:20:07 -05:00
.unwrap();
2025-07-01 17:30:34 -05:00
// dedicated AppSrc helpers exist and avoid the needless `?`
appsrc.set_format(gst::Format::Time);
appsrc.set_is_live(true);
2025-11-30 16:16:03 -03:00
let decodebin = gst::ElementFactory::make("decodebin")
2025-07-01 17:30:34 -05:00
.build()
.context("make decodebin")?;
let convert = gst::ElementFactory::make("audioconvert")
.build()
.context("make audioconvert")?;
let resample = gst::ElementFactory::make("audioresample")
.build()
.context("make audioresample")?;
let caps = gst::Caps::builder("audio/x-raw")
.field("format", "S16LE")
.field("channels", 2i32)
.field("rate", 48_000i32)
.build();
let capsfilter = gst::ElementFactory::make("capsfilter")
.property("caps", &caps)
.build()
.context("make capsfilter")?;
2025-11-30 16:16:03 -03:00
let alsa_sink = gst::ElementFactory::make("alsasink")
2025-07-01 17:30:34 -05:00
.build()
.context("make alsasink")?;
alsa_sink.set_property("device", &alsa_dev);
pipeline.add_many(&[
appsrc.upcast_ref(),
&decodebin,
&convert,
&resample,
&capsfilter,
&alsa_sink,
])?;
2025-07-01 17:30:34 -05:00
appsrc.link(&decodebin)?;
gst::Element::link_many(&[&convert, &resample, &capsfilter, &alsa_sink])?;
2025-07-01 17:30:34 -05:00
/*------------ decodebin autolink ----------------*/
let convert_sink = convert
.static_pad("sink")
.context("audioconvert sink pad")?;
2025-07-01 17:30:34 -05:00
decodebin.connect_pad_added(move |_db, pad| {
if convert_sink.is_linked() {
return;
2025-07-01 17:30:34 -05:00
}
let caps = pad.current_caps().unwrap_or_else(|| pad.query_caps(None));
let is_audio = caps
.structure(0)
.map(|s| s.name().starts_with("audio/"))
.unwrap_or(false);
if !is_audio {
return;
}
let _ = pad.link(&convert_sink);
2025-07-01 17:30:34 -05:00
});
// underrun ≠ error just show a warning
2025-07-01 18:21:06 -05:00
// let _id = alsa_sink.connect("underrun", false, |_| {
// tracing::warn!("⚠️ USB playback underrun host muted/not reading");
// None
// });
2025-07-01 17:30:34 -05:00
pipeline.set_state(gst::State::Playing)?;
Ok(Self {
appsrc,
_pipe: pipeline,
tap: ClipTap::new("voice", Duration::from_secs(60)),
})
}
pub fn push(&mut self, pkt: &AudioPacket) {
self.tap.feed(&pkt.data);
let mut buf = gst::Buffer::from_slice(pkt.data.clone());
buf.get_mut()
.unwrap()
.set_pts(Some(gst::ClockTime::from_useconds(pkt.pts)));
let _ = self.appsrc.push_buffer(buf);
2025-07-01 17:30:34 -05:00
}
pub fn finish(&mut self) {
self.tap.flush();
let _ = self.appsrc.end_of_stream();
2025-06-30 14:20:07 -05:00
}
2025-07-01 17:30:34 -05:00
}