zng_ext_audio/
output.rs

1use std::{fmt, sync::Arc, time::Duration};
2
3use serde::{Deserialize, Serialize};
4use zng_app::{
5    update::UPDATES,
6    view_process::{
7        VIEW_PROCESS, VIEW_PROCESS_INITED_EVENT, ViewAudioOutput,
8        raw_events::{RAW_AUDIO_OUTPUT_OPEN_ERROR_EVENT, RAW_AUDIO_OUTPUT_OPEN_EVENT},
9    },
10};
11use zng_txt::{ToTxt as _, Txt};
12use zng_unit::{Factor, FactorUnits as _};
13use zng_var::{AnyVarHookArgs, Var, impl_from_and_into_var, var};
14use zng_view_api::audio::{AudioMix as ViewAudioMix, AudioMixLayer, AudioOutputConfig, AudioPlayId};
15
16use crate::{AUDIOS, AUDIOS_SV, AudioOutputId, AudioTrack};
17pub use zng_view_api::audio::AudioOutputState;
18
19pub(crate) struct AudioOutputData {
20    id: AudioOutputId,
21    view: Var<Result<ViewAudioOutput, Txt>>,
22
23    volume: Var<Factor>,
24    speed: Var<Factor>,
25    state: Var<AudioOutputState>,
26}
27
28/// Represents an open audio output stream.
29///
30/// You can use [`AUDIOS.open_output`] to open a new output stream.
31///
32/// [`AUDIOS.open_output`]: crate::AUDIOS::open_output
33#[derive(Clone)]
34pub struct AudioOutput(pub(crate) Arc<AudioOutputData>);
35impl fmt::Debug for AudioOutput {
36    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
37        f.debug_struct("AudioOutput")
38            .field("id", &self.0.id)
39            .field("state", &self.0.state.get())
40            .field("volume", &self.0.volume.get())
41            .field("speed", &self.0.speed.get())
42            .finish_non_exhaustive()
43    }
44}
45impl PartialEq for AudioOutput {
46    fn eq(&self, other: &Self) -> bool {
47        self.0.id == other.0.id
48    }
49}
50impl Eq for AudioOutput {}
51impl AudioOutput {
52    pub(crate) fn open(id: AudioOutputId, opt: AudioOutputOptions) -> Self {
53        let r = Self(Arc::new(AudioOutputData {
54            id,
55            view: var(Err(Txt::from("not connected"))),
56            volume: var(opt.config.volume),
57            speed: var(opt.config.speed),
58            state: var(opt.config.state),
59        }));
60        r.0.state.as_any().hook(r.update_view_handler()).perm();
61        r.0.speed.as_any().hook(r.update_view_handler()).perm();
62        r.0.speed.as_any().hook(r.update_view_handler()).perm();
63
64        let handle = RAW_AUDIO_OUTPUT_OPEN_ERROR_EVENT.hook(move |args| {
65            if args.output_id == id {
66                if let Some(o) = AUDIOS_SV.read().outputs.get(&id)
67                    && let Some(o) = o.upgrade()
68                {
69                    o.0.view.set(Err(args.error.clone()));
70                }
71                return false;
72            }
73            true
74        });
75        let handle = RAW_AUDIO_OUTPUT_OPEN_EVENT.hook(move |args| {
76            let _hold = &handle;
77            if args.output_id == id {
78                if let Some(vo) = args.output.upgrade()
79                    && let Some(o) = AUDIOS_SV.read().outputs.get(&id)
80                    && let Some(o) = o.upgrade()
81                {
82                    o.0.view.set(Ok(vo));
83                }
84                return false;
85            }
86            true
87        });
88        let handle = VIEW_PROCESS_INITED_EVENT.hook(move |_| {
89            let _hold = &handle;
90
91            if let Some(o) = AUDIOS_SV.read().outputs.get(&id)
92                && let Some(o) = o.upgrade()
93            {
94                let config = AudioOutputConfig::new(o.0.state.get(), o.0.volume.get(), o.0.speed.get());
95                let _ = VIEW_PROCESS.open_audio_output(zng_view_api::audio::AudioOutputRequest::new(
96                    zng_view_api::audio::AudioOutputId::from_raw(id.get()),
97                    config,
98                ));
99                return true;
100            }
101            false
102        });
103        r.0.view.hold(handle).perm();
104
105        if VIEW_PROCESS.is_connected() {
106            let _ = VIEW_PROCESS.open_audio_output(zng_view_api::audio::AudioOutputRequest::new(
107                zng_view_api::audio::AudioOutputId::from_raw(id.get()),
108                opt.config,
109            ));
110        }
111
112        r
113    }
114    fn update_view_handler(&self) -> impl FnMut(&AnyVarHookArgs) -> bool + Send + 'static {
115        let wk = Arc::downgrade(&self.0);
116        move |_| {
117            if let Some(a) = wk.upgrade() {
118                let r = a.view.with(|v| {
119                    if let Ok(v) = v {
120                        let cfg = AudioOutputConfig::new(a.state.get(), a.volume.get(), a.speed.get());
121                        v.update(cfg)
122                    } else {
123                        Ok(())
124                    }
125                });
126                if let Err(e) = r {
127                    // will reconnect on respawn
128                    a.view.set(Err(e.to_txt()));
129                }
130                true
131            } else {
132                false
133            }
134        }
135    }
136
137    /// Unique ID of this output.
138    pub fn id(&self) -> AudioOutputId {
139        self.0.id
140    }
141
142    /// Enqueue the `audio` for playback in this output.
143    ///
144    /// The audio will play when the output is playing and the previous cued audio finishes.
145    pub fn cue(&self, audio: impl Into<AudioMix>) {
146        self.cue_impl(audio.into());
147    }
148    fn cue_impl(&self, audio: AudioMix) {
149        let s = self.clone();
150        UPDATES.once_update("AudioOutput.cue", move || {
151            let r = s.0.view.with(|v| match v {
152                Ok(v) => v.cue(audio.view),
153                Err(e) => {
154                    tracing::error!("failed to cue audio, {e}");
155                    Ok(AudioPlayId::INVALID)
156                }
157            });
158            if let Err(e) = r {
159                // will reconnect on respawn
160                s.0.view.set(Err(e.to_txt()));
161            }
162        });
163    }
164
165    /// Volume of the sound.
166    ///
167    /// The value multiplies the samples, `1.fct()` is the *natural* volume from the source.
168    pub fn volume(&self) -> Var<Factor> {
169        self.0.volume.clone()
170    }
171
172    /// Speed of the sound.
173    ///
174    /// This is a multiplier of the playback speed and pitch.
175    ///
176    /// * `0.5.fct()` doubles the total duration and halves (lowers) the pitch.
177    /// * `2.fct()` halves the total duration and doubles (raises) the pitch.
178    pub fn speed(&self) -> Var<Factor> {
179        self.0.speed.clone()
180    }
181
182    /// Output playback state.
183    ///
184    /// This variable can be set to change the state.
185    ///
186    /// Note that because variable modifications apply at once you cannot stop and play in the same update cycle using this. Use the
187    ///
188    /// The default value is [`AudioOutputState::Playing`].
189    pub fn state(&self) -> Var<AudioOutputState> {
190        self.0.state.clone()
191    }
192
193    /// Change state to [`Playing`].
194    ///
195    /// Audio is sent to the device for playback as audio is [cued].
196    ///
197    /// [`Playing`]: AudioOutputState::Playing
198    /// [cued]: Self::cue
199    pub fn play(&self) {
200        self.0.state.set(AudioOutputState::Playing);
201    }
202
203    /// Change state to [`Paused`].
204    ///
205    /// Audio playback is paused, cue requests are buffered.
206    ///
207    /// [`Paused`]: AudioOutputState::Paused
208    pub fn pause(&self) {
209        self.0.state.set(AudioOutputState::Paused);
210    }
211
212    /// Change state to [`Stopped`].
213    ///
214    /// Audio playback is paused, all current cue requests are dropped.
215    ///
216    /// [`Stopped`]: AudioOutputState::Stopped
217    pub fn stop(&self) {
218        self.0.state.set(AudioOutputState::Stopped);
219    }
220
221    /// Change state to [`Stopped`] and then [`Playing`] in the same update cycle.
222    ///
223    /// [`Stopped`]: AudioOutputState::Stopped
224    /// [`Playing`]: AudioOutputState::Playing
225    pub fn stop_play(&self) {
226        let s = self.clone();
227        self.0.state.modify(move |a| {
228            a.set(AudioOutputState::Playing);
229            s.0.view.with(|v| {
230                if let Ok(v) = v {
231                    let _ = v.update(AudioOutputConfig::new(AudioOutputState::Stopped, s.0.volume.get(), s.0.speed.get()));
232                    a.update();
233                }
234            });
235        });
236    }
237
238    /// Keep the audio output open until the end of the app.
239    pub fn perm(&self) {
240        AUDIOS.perm_output(self);
241    }
242
243    /// Read-only variable that tracks if this output is connected with the view-process.
244    ///
245    /// When this is `false` the [`error`] might be set. Outputs automatically try to reconnect in case
246    /// of view-process respawn, but note that it will respawn in the [`Stopped`] state.
247    ///
248    /// [`error`]: Self::error
249    /// [`Stopped`]: AudioOutputState::Stopped
250    pub fn is_connected(&self) -> Var<bool> {
251        self.0.view.map(|v| v.is_ok())
252    }
253
254    /// Gets an error message if view-process connection has failed.
255    ///
256    /// Reconnection is attempted on view-process respawn, the [`is_connected`] tracks the ok status. Note that
257    /// the first connection attempt is `true` in [`is_connected`] and `None` here.
258    ///
259    /// [`is_connected`]: Self::is_connected
260    pub fn error(&self) -> Var<Option<Txt>> {
261        self.0.view.map(|v| match v {
262            Ok(_) => None,
263            Err(e) => {
264                if e.is_empty() {
265                    None
266                } else {
267                    Some(e.clone())
268                }
269            }
270        })
271    }
272}
273
274/// Represents an audio source.
275///
276/// Audio is defined by layers, each subsequent layer applies to the computed result of the previous layer.
277#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
278#[non_exhaustive]
279pub struct AudioMix {
280    view: ViewAudioMix,
281}
282impl AudioMix {
283    /// New empty.
284    pub fn new() -> Self {
285        Self { view: ViewAudioMix::new() }
286    }
287
288    /// Plays silence for the `duration` before starting layers mix.
289    pub fn with_delay(mut self, duration: Duration) -> Self {
290        self.view.delay = duration;
291        self
292    }
293
294    /// Set the total duration.
295    ///
296    /// If not set audio plays until all audio and mix layers end. If set audio plays for the duration, if layers end before the duration
297    /// plays silent at the end, if layers exceed the duration they are clipped.
298    ///
299    /// Note that the `duration` must account for the [initial delay], the initial silence is included in the the total duration.
300    ///
301    /// [initial delay]: Self::with_delay
302    pub fn with_total_duration(mut self, duration: Duration) -> Self {
303        self.view.total_duration = Some(duration);
304        self
305    }
306
307    /// Add layer that plays the cached audio.
308    ///
309    /// The audio samples are adapted to the output format and each sample added to the under layers result.
310    pub fn with_audio(mut self, audio: &AudioTrack) -> Self {
311        if !audio.can_cue() {
312            let e = audio.error();
313            tracing::error!(
314                "cannot cue audio, {}",
315                e.as_deref().unwrap_or("metadata not decoded in view-process")
316            );
317            return self;
318        }
319
320        self.view.layers.push(AudioMixLayer::Audio {
321            audio: audio.view_handle().audio_id(),
322            skip: Duration::ZERO,
323            take: Duration::MAX,
324        });
325        self
326    }
327
328    /// Add layer that clips and plays the cached audio.
329    ///
330    /// This is similar to [`with_audio`], but only the range `skip..skip + take` is played.
331    ///
332    /// [`with_audio`]: Self::with_audio
333    pub fn with_audio_clip(mut self, audio: &AudioTrack, skip: Duration, take: Duration) -> Self {
334        if !audio.can_cue() {
335            let e = audio.error();
336            tracing::error!(
337                "cannot cue audio, {}",
338                e.as_deref().unwrap_or("metadata not decoded in view-process")
339            );
340            return self;
341        }
342
343        self.view.layers.push(AudioMixLayer::Audio {
344            audio: audio.view_handle().audio_id(),
345            skip,
346            take,
347        });
348        self
349    }
350
351    /// Add layer that plays another mix.
352    ///
353    /// The inner `mix` is sampled as computed audio, that is, samples are computed first and added to the under layers result.
354    pub fn with_mix(self, mix: impl Into<AudioMix>) -> Self {
355        self.with_mix_clip(mix.into(), Duration::ZERO, Duration::MAX)
356    }
357    /// Add layer that clips and plays another mix.
358    ///
359    /// This is similar to [`with_mix`], but only the range `skip..skip + take` is played.
360    ///
361    /// [`with_mix`]: Self::with_mix
362    pub fn with_mix_clip(mut self, mix: impl Into<AudioMix>, skip: Duration, take: Duration) -> Self {
363        self.view.layers.push(AudioMixLayer::AudioMix {
364            mix: mix.into().view,
365            skip,
366            take,
367        });
368        self
369    }
370    /// Add layer that generates a sine wave sound.
371    ///
372    /// The generated sound samples are added to the under layers result.
373    pub fn with_sine_wave(mut self, frequency: f32, duration: Duration) -> Self {
374        self.view.layers.push(AudioMixLayer::SineWave { frequency, duration });
375        self
376    }
377
378    /// Add effect layer that applies a linear volume transition.
379    ///
380    /// When the playback is in range the computed sample of under layers is multiplied by the linear interpolation
381    /// between `start_volume` and `end_volume`.
382    ///
383    /// Note that outside the volume range is not affected, before and after.
384    pub fn with_volume_linear(mut self, start: Duration, duration: Duration, start_volume: Factor, end_volume: Factor) -> Self {
385        self.view.layers.push(AudioMixLayer::VolumeLinear {
386            start,
387            duration,
388            start_volume,
389            end_volume,
390        });
391        self
392    }
393
394    /// Add an effect layer that fades in from the start over a transition duration.
395    ///
396    /// A linear volume transition at the start raises the volume of under layers from zero to normal over the `transition_duration`.
397    pub fn with_fade_in(self, transition_duration: Duration) -> Self {
398        self.with_volume_linear(Duration::ZERO, transition_duration, 0.fct(), 1.fct())
399    }
400
401    /// Add an effect layer that fades out the audio after `start`.
402    ///
403    /// A linear volume transition lowers the volume of under layers after `start` to zero over `transition_duration`,
404    /// the volume remains zeroed until audio end.
405    ///
406    /// Note that this does not affect the total duration, you must also call [`with_total_duration`] to *fade out and stop*.
407    ///
408    /// [`with_total_duration`]: Self::with_total_duration
409    pub fn with_fade_out(self, start: Duration, transition_duration: Duration) -> Self {
410        self.with_volume_linear(start, transition_duration, 1.fct(), 0.fct())
411            .with_volume_linear(start + transition_duration, Duration::MAX, 0.fct(), 0.fct())
412    }
413}
414impl Default for AudioMix {
415    fn default() -> Self {
416        Self::new()
417    }
418}
419impl_from_and_into_var! {
420    fn from(mix: AudioMix) -> ViewAudioMix {
421        mix.view
422    }
423    fn from(audio: AudioTrack) -> AudioMix {
424        AudioMix::new().with_audio(&audio)
425    }
426}
427impl From<&AudioTrack> for AudioMix {
428    fn from(audio: &AudioTrack) -> Self {
429        AudioMix::new().with_audio(audio)
430    }
431}
432
433/// Options for a new audio output stream.
434#[derive(Debug)]
435#[non_exhaustive]
436pub struct AudioOutputOptions {
437    /// Initial config.
438    pub config: AudioOutputConfig,
439}
440
441impl Default for AudioOutputOptions {
442    fn default() -> Self {
443        Self {
444            config: AudioOutputConfig::new(AudioOutputState::Playing, 1.fct(), 1.fct()),
445        }
446    }
447}
448
449/// Weak reference to an [`AudioOutput`].
450#[derive(Clone)]
451pub struct WeakAudioOutput(std::sync::Weak<AudioOutputData>);
452impl fmt::Debug for WeakAudioOutput {
453    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
454        f.debug_tuple("WeakAudioOutput").finish_non_exhaustive()
455    }
456}
457impl PartialEq for WeakAudioOutput {
458    fn eq(&self, other: &Self) -> bool {
459        self.0.ptr_eq(&other.0)
460    }
461}
462impl Eq for WeakAudioOutput {}
463impl WeakAudioOutput {
464    /// New weak reference that does not allocate and does not upgrade.
465    pub const fn new() -> Self {
466        Self(std::sync::Weak::new())
467    }
468
469    /// Attempt to upgrade to a strong reference to the audio output.
470    pub fn upgrade(&self) -> Option<AudioOutput> {
471        self.0.upgrade().map(AudioOutput)
472    }
473}
474impl Default for WeakAudioOutput {
475    fn default() -> Self {
476        Self::new()
477    }
478}
479
480impl AudioOutput {
481    /// Create a weak reference to this audio output.
482    pub fn downgrade(&self) -> WeakAudioOutput {
483        WeakAudioOutput(std::sync::Arc::downgrade(&self.0))
484    }
485}
zng_ext_audio/output.rs

zng_ext_audio/
output.rs
