mirror of
https://chromium.googlesource.com/crosvm/crosvm
synced 2024-11-24 20:48:55 +00:00
Add virtio-snd device with CRAS backend
Enable with `--cras-snd`. Verified: Basic playback and capture Missing features: * Getting chmap/jack/stream info from CRAS. They are hardcoded for now. * Jack connect/disconnect notifications from CRAS * Reporting latency bytes to the driver. It is currently hardcoded to 0. BUG=b:179757101 TEST=`aplay` and `arecord` inside a debian img with a 5.10 kernel built with virtio snd support. Launched with crosvm on rammus/kukui/hatch Change-Id: I240000a92418b75b3eb8dcd241ff320214b68739 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2777991 Tested-by: kokoro <noreply+kokoro@google.com> Commit-Queue: Woody Chow <woodychow@google.com> Reviewed-by: Chih-Yang Hsia <paulhsia@chromium.org>
This commit is contained in:
parent
bf0294eb7f
commit
737ff125ca
14 changed files with 1371 additions and 39 deletions
20
Cargo.lock
generated
20
Cargo.lock
generated
|
@ -173,6 +173,14 @@ dependencies = [
|
|||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cras-sys"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"audio_streams",
|
||||
"data_model",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.2.1"
|
||||
|
@ -308,9 +316,11 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"acpi_tables",
|
||||
"anyhow",
|
||||
"async-task",
|
||||
"audio_streams",
|
||||
"base",
|
||||
"bit_field",
|
||||
"cras-sys",
|
||||
"cros_async",
|
||||
"data_model",
|
||||
"disk",
|
||||
|
@ -636,6 +646,16 @@ checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"
|
|||
[[package]]
|
||||
name = "libcras"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"audio_streams",
|
||||
"cras-sys",
|
||||
"cros_async",
|
||||
"data_model",
|
||||
"futures",
|
||||
"libc",
|
||||
"sys_util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libcrosvm_control"
|
||||
|
|
|
@ -128,4 +128,4 @@ sys_util = { path = "sys_util" }
|
|||
tempfile = { path = "tempfile" }
|
||||
wire_format_derive = { path = "common/p9/wire_format_derive" } # ignored by ebuild
|
||||
minijail = { path = "third_party/minijail/rust/minijail" } # ignored by ebuild
|
||||
vmm_vhost = { path = "third_party/vmm_vhost", features = ["vhost-user-master", "vhost-user-slave"] } # ignored by ebuild
|
||||
vmm_vhost = { path = "third_party/vmm_vhost", features = ["vhost-user-master", "vhost-user-slave"] } # ignored by ebuild
|
|
@ -19,6 +19,7 @@ virgl_renderer = ["gpu", "rutabaga_gfx/virgl_renderer"]
|
|||
gfxstream = ["gpu", "rutabaga_gfx/gfxstream"]
|
||||
|
||||
[dependencies]
|
||||
async-task = "4"
|
||||
acpi_tables = {path = "../acpi_tables" }
|
||||
anyhow = "*"
|
||||
audio_streams = "*"
|
||||
|
|
|
@ -88,42 +88,42 @@ pub const VIRTIO_SND_PCM_RATE_192000: u8 = 12;
|
|||
pub const VIRTIO_SND_PCM_RATE_384000: u8 = 13;
|
||||
|
||||
/* standard channel position definition */
|
||||
pub const VIRTIO_SND_CHMAP_NONE: u32 = 0; /* undefined */
|
||||
pub const VIRTIO_SND_CHMAP_NA: u32 = 1; /* silent */
|
||||
pub const VIRTIO_SND_CHMAP_MONO: u32 = 2; /* mono stream */
|
||||
pub const VIRTIO_SND_CHMAP_FL: u32 = 3; /* front left */
|
||||
pub const VIRTIO_SND_CHMAP_FR: u32 = 4; /* front right */
|
||||
pub const VIRTIO_SND_CHMAP_RL: u32 = 5; /* rear left */
|
||||
pub const VIRTIO_SND_CHMAP_RR: u32 = 6; /* rear right */
|
||||
pub const VIRTIO_SND_CHMAP_FC: u32 = 7; /* front center */
|
||||
pub const VIRTIO_SND_CHMAP_LFE: u32 = 8; /* low frequency (LFE) */
|
||||
pub const VIRTIO_SND_CHMAP_SL: u32 = 9; /* side left */
|
||||
pub const VIRTIO_SND_CHMAP_SR: u32 = 10; /* side right */
|
||||
pub const VIRTIO_SND_CHMAP_RC: u32 = 11; /* rear center */
|
||||
pub const VIRTIO_SND_CHMAP_FLC: u32 = 12; /* front left center */
|
||||
pub const VIRTIO_SND_CHMAP_FRC: u32 = 13; /* front right center */
|
||||
pub const VIRTIO_SND_CHMAP_RLC: u32 = 14; /* rear left center */
|
||||
pub const VIRTIO_SND_CHMAP_RRC: u32 = 15; /* rear right center */
|
||||
pub const VIRTIO_SND_CHMAP_FLW: u32 = 16; /* front left wide */
|
||||
pub const VIRTIO_SND_CHMAP_FRW: u32 = 17; /* front right wide */
|
||||
pub const VIRTIO_SND_CHMAP_FLH: u32 = 18; /* front left high */
|
||||
pub const VIRTIO_SND_CHMAP_FCH: u32 = 19; /* front center high */
|
||||
pub const VIRTIO_SND_CHMAP_FRH: u32 = 20; /* front right high */
|
||||
pub const VIRTIO_SND_CHMAP_TC: u32 = 21; /* top center */
|
||||
pub const VIRTIO_SND_CHMAP_TFL: u32 = 22; /* top front left */
|
||||
pub const VIRTIO_SND_CHMAP_TFR: u32 = 23; /* top front right */
|
||||
pub const VIRTIO_SND_CHMAP_TFC: u32 = 24; /* top front center */
|
||||
pub const VIRTIO_SND_CHMAP_TRL: u32 = 25; /* top rear left */
|
||||
pub const VIRTIO_SND_CHMAP_TRR: u32 = 26; /* top rear right */
|
||||
pub const VIRTIO_SND_CHMAP_TRC: u32 = 27; /* top rear center */
|
||||
pub const VIRTIO_SND_CHMAP_TFLC: u32 = 28; /* top front left center */
|
||||
pub const VIRTIO_SND_CHMAP_TFRC: u32 = 29; /* top front right center */
|
||||
pub const VIRTIO_SND_CHMAP_TSL: u32 = 34; /* top side left */
|
||||
pub const VIRTIO_SND_CHMAP_TSR: u32 = 35; /* top side right */
|
||||
pub const VIRTIO_SND_CHMAP_LLFE: u32 = 36; /* left LFE */
|
||||
pub const VIRTIO_SND_CHMAP_RLFE: u32 = 37; /* right LFE */
|
||||
pub const VIRTIO_SND_CHMAP_BC: u32 = 38; /* bottom center */
|
||||
pub const VIRTIO_SND_CHMAP_BLC: u32 = 39; /* bottom left center */
|
||||
pub const VIRTIO_SND_CHMAP_BRC: u32 = 40; /* bottom right center */
|
||||
pub const VIRTIO_SND_CHMAP_NONE: u8 = 0; /* undefined */
|
||||
pub const VIRTIO_SND_CHMAP_NA: u8 = 1; /* silent */
|
||||
pub const VIRTIO_SND_CHMAP_MONO: u8 = 2; /* mono stream */
|
||||
pub const VIRTIO_SND_CHMAP_FL: u8 = 3; /* front left */
|
||||
pub const VIRTIO_SND_CHMAP_FR: u8 = 4; /* front right */
|
||||
pub const VIRTIO_SND_CHMAP_RL: u8 = 5; /* rear left */
|
||||
pub const VIRTIO_SND_CHMAP_RR: u8 = 6; /* rear right */
|
||||
pub const VIRTIO_SND_CHMAP_FC: u8 = 7; /* front center */
|
||||
pub const VIRTIO_SND_CHMAP_LFE: u8 = 8; /* low frequency (LFE) */
|
||||
pub const VIRTIO_SND_CHMAP_SL: u8 = 9; /* side left */
|
||||
pub const VIRTIO_SND_CHMAP_SR: u8 = 10; /* side right */
|
||||
pub const VIRTIO_SND_CHMAP_RC: u8 = 11; /* rear center */
|
||||
pub const VIRTIO_SND_CHMAP_FLC: u8 = 12; /* front left center */
|
||||
pub const VIRTIO_SND_CHMAP_FRC: u8 = 13; /* front right center */
|
||||
pub const VIRTIO_SND_CHMAP_RLC: u8 = 14; /* rear left center */
|
||||
pub const VIRTIO_SND_CHMAP_RRC: u8 = 15; /* rear right center */
|
||||
pub const VIRTIO_SND_CHMAP_FLW: u8 = 16; /* front left wide */
|
||||
pub const VIRTIO_SND_CHMAP_FRW: u8 = 17; /* front right wide */
|
||||
pub const VIRTIO_SND_CHMAP_FLH: u8 = 18; /* front left high */
|
||||
pub const VIRTIO_SND_CHMAP_FCH: u8 = 19; /* front center high */
|
||||
pub const VIRTIO_SND_CHMAP_FRH: u8 = 20; /* front right high */
|
||||
pub const VIRTIO_SND_CHMAP_TC: u8 = 21; /* top center */
|
||||
pub const VIRTIO_SND_CHMAP_TFL: u8 = 22; /* top front left */
|
||||
pub const VIRTIO_SND_CHMAP_TFR: u8 = 23; /* top front right */
|
||||
pub const VIRTIO_SND_CHMAP_TFC: u8 = 24; /* top front center */
|
||||
pub const VIRTIO_SND_CHMAP_TRL: u8 = 25; /* top rear left */
|
||||
pub const VIRTIO_SND_CHMAP_TRR: u8 = 26; /* top rear right */
|
||||
pub const VIRTIO_SND_CHMAP_TRC: u8 = 27; /* top rear center */
|
||||
pub const VIRTIO_SND_CHMAP_TFLC: u8 = 28; /* top front left center */
|
||||
pub const VIRTIO_SND_CHMAP_TFRC: u8 = 29; /* top front right center */
|
||||
pub const VIRTIO_SND_CHMAP_TSL: u8 = 34; /* top side left */
|
||||
pub const VIRTIO_SND_CHMAP_TSR: u8 = 35; /* top side right */
|
||||
pub const VIRTIO_SND_CHMAP_LLFE: u8 = 36; /* left LFE */
|
||||
pub const VIRTIO_SND_CHMAP_RLFE: u8 = 37; /* right LFE */
|
||||
pub const VIRTIO_SND_CHMAP_BC: u8 = 38; /* bottom center */
|
||||
pub const VIRTIO_SND_CHMAP_BLC: u8 = 39; /* bottom left center */
|
||||
pub const VIRTIO_SND_CHMAP_BRC: u8 = 40; /* bottom right center */
|
||||
|
||||
pub const VIRTIO_SND_CHMAP_MAX_SIZE: usize = 18;
|
||||
|
|
613
devices/src/virtio/snd/cras_backend/async_funcs.rs
Normal file
613
devices/src/virtio/snd/cras_backend/async_funcs.rs
Normal file
|
@ -0,0 +1,613 @@
|
|||
// Copyright 2021 The Chromium OS Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
use futures::{channel::mpsc, SinkExt, StreamExt};
|
||||
use std::io::{self, Write};
|
||||
use std::rc::Rc;
|
||||
|
||||
use base::error;
|
||||
use cros_async::{sync::Condvar, sync::Mutex as AsyncMutex, EventAsync, Executor};
|
||||
use data_model::{DataInit, Le32};
|
||||
use vm_memory::GuestMemory;
|
||||
|
||||
use crate::virtio::snd::common::*;
|
||||
use crate::virtio::snd::constants::*;
|
||||
use crate::virtio::snd::layout::*;
|
||||
use crate::virtio::{DescriptorChain, Interrupt, Queue, Reader, Writer};
|
||||
|
||||
use super::{DirectionalStream, Error, SndData, StreamInfo, WorkerStatus};
|
||||
|
||||
// Returns true if the operation is successful. Returns error if there is
|
||||
// a runtime/internal error
|
||||
async fn process_pcm_ctrl(
|
||||
ex: &Executor,
|
||||
mem: &GuestMemory,
|
||||
tx_queue: &Rc<AsyncMutex<Queue>>,
|
||||
rx_queue: &Rc<AsyncMutex<Queue>>,
|
||||
interrupt: &Rc<Interrupt>,
|
||||
streams: &Rc<AsyncMutex<Vec<AsyncMutex<StreamInfo<'_>>>>>,
|
||||
cmd_code: u32,
|
||||
writer: &mut Writer,
|
||||
stream_id: usize,
|
||||
) -> Result<(), Error> {
|
||||
let streams = streams.read_lock().await;
|
||||
let mut stream = match streams.get(stream_id) {
|
||||
Some(stream_info) => stream_info.lock().await,
|
||||
None => {
|
||||
error!(
|
||||
"Stream id={} not found for {}. Error code: VIRTIO_SND_S_BAD_MSG",
|
||||
stream_id,
|
||||
get_virtio_snd_r_pcm_cmd_name(cmd_code)
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_BAD_MSG)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
};
|
||||
|
||||
let result = match cmd_code {
|
||||
VIRTIO_SND_R_PCM_PREPARE => {
|
||||
stream
|
||||
.prepare(ex, mem.clone(), tx_queue, rx_queue, interrupt)
|
||||
.await
|
||||
}
|
||||
VIRTIO_SND_R_PCM_START => stream.start().await,
|
||||
VIRTIO_SND_R_PCM_STOP => stream.stop().await,
|
||||
VIRTIO_SND_R_PCM_RELEASE => stream.release().await,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
match result {
|
||||
Ok(_) => {
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_OK)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
Err(Error::OperationNotSupported) => {
|
||||
error!(
|
||||
"{} for stream id={} failed. Error code: VIRTIO_SND_S_NOT_SUPP.",
|
||||
get_virtio_snd_r_pcm_cmd_name(cmd_code),
|
||||
stream_id
|
||||
);
|
||||
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_NOT_SUPP)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
Err(e) => {
|
||||
// Runtime/internal error would be more appropriate, but there's
|
||||
// no such error type
|
||||
error!(
|
||||
"{} for stream id={} failed. Error code: VIRTIO_SND_S_IO_ERR. Actual error: {}",
|
||||
get_virtio_snd_r_pcm_cmd_name(cmd_code),
|
||||
stream_id,
|
||||
e
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_IO_ERR)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Start a pcm worker that receives descriptors containing PCM frames (audio data) from the tx/rx
|
||||
/// queue, and forward them to CRAS. One pcm worker is needed per stream.
|
||||
pub async fn start_pcm_worker(
|
||||
ex: Executor,
|
||||
mut dstream: DirectionalStream,
|
||||
mut desc_receiver: mpsc::UnboundedReceiver<DescriptorChain>,
|
||||
status_mutex: Rc<AsyncMutex<WorkerStatus>>,
|
||||
cv: Rc<Condvar>,
|
||||
mem: GuestMemory,
|
||||
queue: Rc<AsyncMutex<Queue>>,
|
||||
interrupt: Rc<Interrupt>,
|
||||
period_bytes: usize,
|
||||
) -> Result<(), Error> {
|
||||
loop {
|
||||
let mut o_desc_chain = desc_receiver.next().await;
|
||||
{
|
||||
let mut status = status_mutex.lock().await;
|
||||
while *status == WorkerStatus::Pause {
|
||||
// async wait
|
||||
status = cv.wait(status).await;
|
||||
}
|
||||
if *status == WorkerStatus::Quit {
|
||||
while let Some(desc_chain) = o_desc_chain {
|
||||
// From the virtio-snd spec:
|
||||
// The device MUST complete all pending I/O messages for the specified stream ID.
|
||||
send_pcm_response(
|
||||
&mem,
|
||||
desc_chain,
|
||||
&queue,
|
||||
&interrupt,
|
||||
virtio_snd_pcm_status {
|
||||
status: Le32::from(VIRTIO_SND_S_OK),
|
||||
latency_bytes: Le32::from(0),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
o_desc_chain = desc_receiver.next().await
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
let desc_chain =
|
||||
o_desc_chain.expect("Unreachable. status should be Quit when the channel is closed");
|
||||
|
||||
let index = desc_chain.index;
|
||||
|
||||
let mut reader =
|
||||
Reader::new(mem.clone(), desc_chain.clone()).map_err(Error::DescriptorChain)?;
|
||||
let mut writer = Writer::new(mem.clone(), desc_chain).map_err(Error::DescriptorChain)?;
|
||||
|
||||
let copy_data = async {
|
||||
// stream_id was already read in handle_pcm_worker
|
||||
reader.consume(std::mem::size_of::<virtio_snd_pcm_xfer>());
|
||||
|
||||
// TODO: How to check wrong direction?
|
||||
// return Err(Error::StreamWrongDirection);
|
||||
match &mut dstream {
|
||||
DirectionalStream::Output(stream) => {
|
||||
let mut dst_buf = stream
|
||||
.next_playback_buffer(&ex)
|
||||
.await
|
||||
.map_err(Error::FetchBuffer)?;
|
||||
let transferred = io::copy(&mut reader, &mut dst_buf).map_err(Error::Io)?;
|
||||
if transferred as usize != period_bytes {
|
||||
error!(
|
||||
"Bytes written {} != period_bytes {}",
|
||||
transferred, period_bytes
|
||||
);
|
||||
return Err(Error::InvalidBufferSize);
|
||||
}
|
||||
dst_buf.commit().await;
|
||||
}
|
||||
DirectionalStream::Input(stream) => {
|
||||
let mut src_buf = stream
|
||||
.next_capture_buffer(&ex)
|
||||
.await
|
||||
.map_err(Error::FetchBuffer)?;
|
||||
|
||||
let transferred = io::copy(&mut src_buf, &mut writer).map_err(Error::Io)?;
|
||||
if transferred as usize != period_bytes {
|
||||
error!(
|
||||
"Bytes written {} != period_bytes {}",
|
||||
transferred, period_bytes
|
||||
);
|
||||
return Err(Error::InvalidBufferSize);
|
||||
}
|
||||
src_buf.commit().await;
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
};
|
||||
|
||||
// From the spec: VIRTIO_SND_S_OK if an operation is successful, and
|
||||
// VIRTIO_SND_S_IO_ERR otherwise.
|
||||
let status = match copy_data.await {
|
||||
Ok(()) => VIRTIO_SND_S_OK,
|
||||
Err(e) => {
|
||||
error!("PCM I/O message failed: {}", e);
|
||||
VIRTIO_SND_S_IO_ERR
|
||||
}
|
||||
};
|
||||
|
||||
send_pcm_response_with_writer(
|
||||
writer,
|
||||
index,
|
||||
&mem,
|
||||
&queue,
|
||||
&interrupt,
|
||||
// TODO(woodychow): Extend audio_streams API, and fetch latency_bytes from
|
||||
// `next_playback_buffer` or `next_capture_buffer`"
|
||||
virtio_snd_pcm_status {
|
||||
status: Le32::from(status),
|
||||
latency_bytes: Le32::from(0),
|
||||
},
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_pcm_response_with_writer(
|
||||
mut writer: Writer,
|
||||
desc_index: u16,
|
||||
mem: &GuestMemory,
|
||||
rc_queue: &Rc<AsyncMutex<Queue>>,
|
||||
interrupt: &Rc<Interrupt>,
|
||||
status: virtio_snd_pcm_status,
|
||||
) -> Result<(), Error> {
|
||||
// For rx queue only. Fast forward the unused audio data buffer.
|
||||
if writer.available_bytes() > std::mem::size_of::<virtio_snd_pcm_status>() {
|
||||
writer
|
||||
.consume_bytes(writer.available_bytes() - std::mem::size_of::<virtio_snd_pcm_status>());
|
||||
}
|
||||
writer.write_obj(status).map_err(Error::WriteResponse)?;
|
||||
let mut queue = rc_queue.lock().await;
|
||||
queue.add_used(mem, desc_index, writer.bytes_written() as u32);
|
||||
queue.trigger_interrupt(mem, &**interrupt);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_pcm_response(
|
||||
mem: &GuestMemory,
|
||||
desc_chain: DescriptorChain,
|
||||
rc_queue: &Rc<AsyncMutex<Queue>>,
|
||||
interrupt: &Rc<Interrupt>,
|
||||
status: virtio_snd_pcm_status,
|
||||
) -> Result<(), Error> {
|
||||
let index = desc_chain.index;
|
||||
let writer = Writer::new(mem.clone(), desc_chain).map_err(Error::DescriptorChain)?;
|
||||
send_pcm_response_with_writer(writer, index, mem, rc_queue, interrupt, status).await
|
||||
}
|
||||
|
||||
/// Handle messages from the tx or the rx queue. One invocation is needed for
|
||||
/// each queue.
|
||||
pub async fn handle_pcm_queue<'a>(
|
||||
mem: &GuestMemory,
|
||||
streams: &Rc<AsyncMutex<Vec<AsyncMutex<StreamInfo<'a>>>>>,
|
||||
queue: &Rc<AsyncMutex<Queue>>,
|
||||
queue_event: EventAsync,
|
||||
interrupt: &Rc<Interrupt>,
|
||||
) -> Result<(), Error> {
|
||||
loop {
|
||||
// Manual queue.next_async() to avoid holding the mutex
|
||||
let foo = async {
|
||||
loop {
|
||||
// Check if there are more descriptors available.
|
||||
if let Some(chain) = queue.lock().await.pop(mem) {
|
||||
return Ok(chain);
|
||||
}
|
||||
queue_event.next_val().await?;
|
||||
}
|
||||
};
|
||||
let desc_chain = foo.await.map_err(Error::Async)?;
|
||||
|
||||
let mut reader =
|
||||
Reader::new(mem.clone(), desc_chain.clone()).map_err(Error::DescriptorChain)?;
|
||||
|
||||
let pcm_xfer: virtio_snd_pcm_xfer = reader.read_obj().map_err(Error::ReadMessage)?;
|
||||
let stream_id: usize = u32::from(pcm_xfer.stream_id) as usize;
|
||||
|
||||
let streams = streams.read_lock().await;
|
||||
let stream_info = match streams.get(stream_id) {
|
||||
Some(stream_info) => stream_info.read_lock().await,
|
||||
None => {
|
||||
error!(
|
||||
"stream_id ({}) < num_streams ({})",
|
||||
stream_id,
|
||||
streams.len()
|
||||
);
|
||||
send_pcm_response(
|
||||
mem,
|
||||
desc_chain,
|
||||
queue,
|
||||
interrupt,
|
||||
virtio_snd_pcm_status {
|
||||
status: Le32::from(VIRTIO_SND_S_IO_ERR),
|
||||
latency_bytes: Le32::from(0),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match stream_info.sender.as_ref() {
|
||||
Some(mut s) => {
|
||||
s.send(desc_chain).await.map_err(Error::MpscRead)?;
|
||||
}
|
||||
None => {
|
||||
send_pcm_response(
|
||||
mem,
|
||||
desc_chain,
|
||||
queue,
|
||||
interrupt,
|
||||
virtio_snd_pcm_status {
|
||||
status: Le32::from(VIRTIO_SND_S_IO_ERR),
|
||||
latency_bytes: Le32::from(0),
|
||||
},
|
||||
)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle all the control messages from the ctrl queue.
|
||||
pub async fn handle_ctrl_queue(
|
||||
ex: &Executor,
|
||||
mem: &GuestMemory,
|
||||
streams: &Rc<AsyncMutex<Vec<AsyncMutex<StreamInfo<'_>>>>>,
|
||||
snd_data: &SndData,
|
||||
mut queue: Queue,
|
||||
mut queue_event: EventAsync,
|
||||
interrupt: &Rc<Interrupt>,
|
||||
tx_queue: &Rc<AsyncMutex<Queue>>,
|
||||
rx_queue: &Rc<AsyncMutex<Queue>>,
|
||||
) -> Result<(), Error> {
|
||||
loop {
|
||||
let desc_chain = queue
|
||||
.next_async(mem, &mut queue_event)
|
||||
.await
|
||||
.map_err(Error::Async)?;
|
||||
|
||||
let index = desc_chain.index;
|
||||
|
||||
let mut reader =
|
||||
Reader::new(mem.clone(), desc_chain.clone()).map_err(Error::DescriptorChain)?;
|
||||
let mut writer = Writer::new(mem.clone(), desc_chain).map_err(Error::DescriptorChain)?;
|
||||
// Don't advance the reader
|
||||
let code = reader
|
||||
.clone()
|
||||
.read_obj::<virtio_snd_hdr>()
|
||||
.map_err(Error::ReadMessage)?
|
||||
.code
|
||||
.into();
|
||||
|
||||
let handle_ctrl_msg = async {
|
||||
return match code {
|
||||
VIRTIO_SND_R_JACK_INFO => {
|
||||
let query_info: virtio_snd_query_info =
|
||||
reader.read_obj().map_err(Error::ReadMessage)?;
|
||||
let start_id: usize = u32::from(query_info.start_id) as usize;
|
||||
let count: usize = u32::from(query_info.count) as usize;
|
||||
if start_id + count > snd_data.jack_info.len() {
|
||||
error!(
|
||||
"start_id({}) + count({}) must be smaller than the number of jacks ({})",
|
||||
start_id,
|
||||
count,
|
||||
snd_data.jack_info.len()
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_BAD_MSG)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
// The response consists of the virtio_snd_hdr structure (contains the request
|
||||
// status code), followed by the device-writable information structures of the
|
||||
// item. Each information structure begins with the following common header
|
||||
writer
|
||||
.write_obj(VIRTIO_SND_S_OK)
|
||||
.map_err(Error::WriteResponse)?;
|
||||
for i in start_id..(start_id + count) {
|
||||
writer
|
||||
.write_all(snd_data.jack_info[i].as_slice())
|
||||
.map_err(Error::WriteResponse)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VIRTIO_SND_R_PCM_INFO => {
|
||||
let query_info: virtio_snd_query_info =
|
||||
reader.read_obj().map_err(Error::ReadMessage)?;
|
||||
let start_id: usize = u32::from(query_info.start_id) as usize;
|
||||
let count: usize = u32::from(query_info.count) as usize;
|
||||
if start_id + count > snd_data.pcm_info.len() {
|
||||
error!(
|
||||
"start_id({}) + count({}) must be smaller than the number of streams ({})",
|
||||
start_id,
|
||||
count,
|
||||
snd_data.pcm_info.len()
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_BAD_MSG)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
// The response consists of the virtio_snd_hdr structure (contains the request
|
||||
// status code), followed by the device-writable information structures of the
|
||||
// item. Each information structure begins with the following common header
|
||||
writer
|
||||
.write_obj(VIRTIO_SND_S_OK)
|
||||
.map_err(Error::WriteResponse)?;
|
||||
for i in start_id..(start_id + count) {
|
||||
writer
|
||||
.write_all(snd_data.pcm_info[i].as_slice())
|
||||
.map_err(Error::WriteResponse)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VIRTIO_SND_R_CHMAP_INFO => {
|
||||
let query_info: virtio_snd_query_info =
|
||||
reader.read_obj().map_err(Error::ReadMessage)?;
|
||||
let start_id: usize = u32::from(query_info.start_id) as usize;
|
||||
let count: usize = u32::from(query_info.count) as usize;
|
||||
if start_id + count > snd_data.chmap_info.len() {
|
||||
error!(
|
||||
"start_id({}) + count({}) must be smaller than the number of chmaps ({})",
|
||||
start_id,
|
||||
count,
|
||||
snd_data.pcm_info.len()
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_BAD_MSG)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
// The response consists of the virtio_snd_hdr structure (contains the request
|
||||
// status code), followed by the device-writable information structures of the
|
||||
// item. Each information structure begins with the following common header
|
||||
writer
|
||||
.write_obj(VIRTIO_SND_S_OK)
|
||||
.map_err(Error::WriteResponse)?;
|
||||
for i in start_id..(start_id + count) {
|
||||
writer
|
||||
.write_all(snd_data.chmap_info[i].as_slice())
|
||||
.map_err(Error::WriteResponse)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
VIRTIO_SND_R_JACK_REMAP => {
|
||||
unreachable!("remap is unsupported");
|
||||
}
|
||||
VIRTIO_SND_R_PCM_SET_PARAMS => {
|
||||
// Raise VIRTIO_SND_S_BAD_MSG or IO error?
|
||||
let set_params: virtio_snd_pcm_set_params =
|
||||
reader.read_obj().map_err(Error::ReadMessage)?;
|
||||
let stream_id: usize = u32::from(set_params.hdr.stream_id) as usize;
|
||||
let buffer_bytes: u32 = set_params.buffer_bytes.into();
|
||||
let period_bytes: u32 = set_params.period_bytes.into();
|
||||
|
||||
let dir = match snd_data.pcm_info.get(stream_id) {
|
||||
Some(pcm_info) => {
|
||||
if set_params.channels < pcm_info.channels_min
|
||||
|| set_params.channels > pcm_info.channels_max
|
||||
{
|
||||
error!(
|
||||
"Number of channels ({}) must be between {} and {}",
|
||||
set_params.channels,
|
||||
pcm_info.channels_min,
|
||||
pcm_info.channels_max
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_NOT_SUPP)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
if (u64::from(pcm_info.formats) & (1 << set_params.format)) == 0 {
|
||||
error!("PCM format {} is not supported.", set_params.format);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_NOT_SUPP)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
if (u64::from(pcm_info.rates) & (1 << set_params.rate)) == 0 {
|
||||
error!("PCM frame rate {} is not supported.", set_params.rate);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_NOT_SUPP)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
|
||||
pcm_info.direction
|
||||
}
|
||||
None => {
|
||||
error!(
|
||||
"stream_id {} < streams {}",
|
||||
stream_id,
|
||||
snd_data.pcm_info.len()
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_BAD_MSG)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
};
|
||||
|
||||
if set_params.features != 0 {
|
||||
error!("No feature is supported");
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_NOT_SUPP)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
|
||||
if buffer_bytes % period_bytes != 0 {
|
||||
error!(
|
||||
"buffer_bytes({}) must be dividable by period_bytes({})",
|
||||
buffer_bytes, period_bytes
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_BAD_MSG)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
|
||||
let streams = streams.read_lock().await;
|
||||
let mut stream_info = match streams.get(stream_id) {
|
||||
Some(stream_info) => stream_info.lock().await,
|
||||
None => {
|
||||
error!("stream_id {} < streams {}", stream_id, streams.len());
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_BAD_MSG)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
};
|
||||
|
||||
if stream_info.state != 0
|
||||
&& stream_info.state != VIRTIO_SND_R_PCM_SET_PARAMS
|
||||
&& stream_info.state != VIRTIO_SND_R_PCM_PREPARE
|
||||
&& stream_info.state != VIRTIO_SND_R_PCM_RELEASE
|
||||
{
|
||||
error!(
|
||||
"Invalid PCM state transition from {} to {}",
|
||||
get_virtio_snd_r_pcm_cmd_name(stream_info.state),
|
||||
get_virtio_snd_r_pcm_cmd_name(VIRTIO_SND_R_PCM_SET_PARAMS)
|
||||
);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_NOT_SUPP)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
|
||||
// Only required for PREPARE -> SET_PARAMS
|
||||
stream_info.release_worker().await?;
|
||||
|
||||
stream_info.channels = set_params.channels;
|
||||
stream_info.format = from_virtio_sample_format(set_params.format).unwrap();
|
||||
stream_info.frame_rate = from_virtio_frame_rate(set_params.rate).unwrap();
|
||||
stream_info.buffer_bytes = buffer_bytes as usize;
|
||||
stream_info.period_bytes = period_bytes as usize;
|
||||
stream_info.direction = dir;
|
||||
stream_info.state = VIRTIO_SND_R_PCM_SET_PARAMS;
|
||||
|
||||
writer
|
||||
.write_obj(VIRTIO_SND_S_OK)
|
||||
.map_err(Error::WriteResponse)
|
||||
}
|
||||
VIRTIO_SND_R_PCM_PREPARE
|
||||
| VIRTIO_SND_R_PCM_START
|
||||
| VIRTIO_SND_R_PCM_STOP
|
||||
| VIRTIO_SND_R_PCM_RELEASE => {
|
||||
let hdr: virtio_snd_pcm_hdr = reader.read_obj().map_err(Error::ReadMessage)?;
|
||||
let stream_id: usize = u32::from(hdr.stream_id) as usize;
|
||||
process_pcm_ctrl(
|
||||
ex,
|
||||
&mem.clone(),
|
||||
tx_queue,
|
||||
rx_queue,
|
||||
interrupt,
|
||||
streams,
|
||||
code,
|
||||
&mut writer,
|
||||
stream_id,
|
||||
)
|
||||
.await
|
||||
.and(Ok(()))?;
|
||||
Ok(())
|
||||
}
|
||||
c => {
|
||||
error!("Unrecognized code: {}", c);
|
||||
return writer
|
||||
.write_obj(VIRTIO_SND_S_BAD_MSG)
|
||||
.map_err(Error::WriteResponse);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
handle_ctrl_msg.await?;
|
||||
queue.add_used(mem, index, writer.bytes_written() as u32);
|
||||
queue.trigger_interrupt(&mem, &**interrupt);
|
||||
}
|
||||
}
|
||||
|
||||
/// Send events to the audio driver.
|
||||
pub async fn handle_event_queue(
|
||||
mem: &GuestMemory,
|
||||
mut queue: Queue,
|
||||
mut queue_event: EventAsync,
|
||||
interrupt: &Rc<Interrupt>,
|
||||
) -> Result<(), Error> {
|
||||
loop {
|
||||
let desc_chain = queue
|
||||
.next_async(mem, &mut queue_event)
|
||||
.await
|
||||
.map_err(Error::Async)?;
|
||||
|
||||
// TODO(woodychow): Poll and forward events from cras asynchronously (API to be added)
|
||||
let index = desc_chain.index;
|
||||
queue.add_used(mem, index, 0);
|
||||
queue.trigger_interrupt(&mem, &**interrupt);
|
||||
}
|
||||
}
|
||||
|
||||
// Async task that waits for a signal from the kill event given to the device at startup. Once this event is
|
||||
// readable, exit. Exiting this future will cause the main loop to break and the worker thread to
|
||||
// exit.
|
||||
pub async fn wait_kill(kill_evt: EventAsync) {
|
||||
let _ = kill_evt.next_val().await;
|
||||
}
|
606
devices/src/virtio/snd/cras_backend/mod.rs
Normal file
606
devices/src/virtio/snd/cras_backend/mod.rs
Normal file
|
@ -0,0 +1,606 @@
|
|||
// Copyright 2021 The Chromium OS Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// virtio-sound spec: https://github.com/oasis-tcs/virtio-spec/blob/master/virtio-sound.tex
|
||||
|
||||
use std::io;
|
||||
use std::rc::Rc;
|
||||
use std::thread;
|
||||
|
||||
use audio_streams::{SampleFormat, StreamSource};
|
||||
use base::{error, warn, Error as SysError, Event, RawDescriptor};
|
||||
use cros_async::sync::{Condvar, Mutex as AsyncMutex};
|
||||
use cros_async::{select4, AsyncError, EventAsync, Executor, SelectResult};
|
||||
use data_model::DataInit;
|
||||
use futures::channel::mpsc;
|
||||
use futures::{pin_mut, Future, TryFutureExt};
|
||||
use libcras::{BoxError, CrasClient, CrasClientType};
|
||||
use thiserror::Error as ThisError;
|
||||
use vm_memory::GuestMemory;
|
||||
|
||||
use crate::virtio::snd::common::*;
|
||||
use crate::virtio::snd::constants::*;
|
||||
use crate::virtio::snd::layout::*;
|
||||
use crate::virtio::{
|
||||
copy_config, DescriptorChain, DescriptorError, Interrupt, Queue, VirtioDevice, TYPE_SOUND,
|
||||
};
|
||||
|
||||
pub mod async_funcs;
|
||||
use crate::virtio::snd::cras_backend::async_funcs::*;
|
||||
|
||||
// control + event + tx + rx queue
|
||||
const NUM_QUEUES: usize = 4;
|
||||
const QUEUE_SIZE: u16 = 1024;
|
||||
|
||||
#[derive(ThisError, Debug)]
|
||||
pub enum Error {
|
||||
/// next_async failed.
|
||||
#[error("Failed to read descriptor asynchronously: {0}")]
|
||||
Async(AsyncError),
|
||||
/// Creating stream failed.
|
||||
#[error("Failed to create stream: {0}")]
|
||||
CreateStream(BoxError),
|
||||
/// Creating kill event failed.
|
||||
#[error("Failed to create kill event: {0}")]
|
||||
CreateKillEvent(SysError),
|
||||
/// Creating WaitContext failed.
|
||||
#[error("Failed to create wait context: {0}")]
|
||||
CreateWaitContext(SysError),
|
||||
/// Cloning kill event failed.
|
||||
#[error("Failed to clone kill event: {0}")]
|
||||
CloneKillEvent(SysError),
|
||||
/// Descriptor chain was invalid.
|
||||
#[error("Failed to valildate descriptor chain: {0}")]
|
||||
DescriptorChain(DescriptorError),
|
||||
/// Error reading message from queue.
|
||||
#[error("Failed to read message: {0}")]
|
||||
ReadMessage(io::Error),
|
||||
/// Failed writing a response to a control message.
|
||||
#[error("Failed to write message response: {0}")]
|
||||
WriteResponse(io::Error),
|
||||
/// Libcras error.
|
||||
#[error("Error in libcras: {0}")]
|
||||
Libcras(libcras::Error),
|
||||
// Mpsc read error.
|
||||
#[error("Error in mpsc: {0}")]
|
||||
MpscRead(futures::channel::mpsc::SendError),
|
||||
/// Stream not found.
|
||||
#[error("stream id ({0}) < num_streams ({1})")]
|
||||
StreamNotFound(usize, usize),
|
||||
/// Fetch buffer error
|
||||
#[error("Failed to get buffer from CRAS: {0}")]
|
||||
FetchBuffer(BoxError),
|
||||
/// Invalid buffer size
|
||||
#[error("Invalid buffer size")]
|
||||
InvalidBufferSize,
|
||||
/// IoError
|
||||
#[error("I/O failed: {0}")]
|
||||
Io(io::Error),
|
||||
/// Operation not supported.
|
||||
#[error("Operation not supported")]
|
||||
OperationNotSupported,
|
||||
/// Writing to a buffer in the guest failed.
|
||||
#[error("failed to write to buffer: {0}")]
|
||||
WriteBuffer(io::Error),
|
||||
}
|
||||
|
||||
pub enum DirectionalStream {
|
||||
Input(Box<dyn audio_streams::capture::AsyncCaptureBufferStream>),
|
||||
Output(Box<dyn audio_streams::AsyncPlaybackBufferStream>),
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, std::cmp::PartialEq)]
|
||||
pub enum WorkerStatus {
|
||||
Pause = 0,
|
||||
Running = 1,
|
||||
Quit = 2,
|
||||
}
|
||||
pub struct StreamInfo<'a> {
|
||||
client: Option<CrasClient<'a>>,
|
||||
channels: u8,
|
||||
format: SampleFormat,
|
||||
frame_rate: u32,
|
||||
buffer_bytes: usize,
|
||||
period_bytes: usize,
|
||||
direction: u8,
|
||||
state: u32, // VIRTIO_SND_R_PCM_SET_PARAMS -> VIRTIO_SND_R_PCM_STOP, or 0 (uninitialized)
|
||||
|
||||
// Worker related
|
||||
status_mutex: Rc<AsyncMutex<WorkerStatus>>,
|
||||
cv: Rc<Condvar>,
|
||||
sender: Option<mpsc::UnboundedSender<DescriptorChain>>,
|
||||
worker_future: Option<Box<dyn Future<Output = Result<(), Error>> + Unpin>>,
|
||||
}
|
||||
|
||||
impl Default for StreamInfo<'_> {
|
||||
fn default() -> Self {
|
||||
StreamInfo {
|
||||
client: None,
|
||||
channels: 0,
|
||||
format: SampleFormat::U8,
|
||||
frame_rate: 0,
|
||||
buffer_bytes: 0,
|
||||
period_bytes: 0,
|
||||
direction: 0,
|
||||
state: 0,
|
||||
status_mutex: Rc::new(AsyncMutex::new(WorkerStatus::Pause)),
|
||||
cv: Rc::new(Condvar::new()),
|
||||
sender: None,
|
||||
worker_future: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stores constant data
|
||||
pub struct SndData {
|
||||
jack_info: Vec<virtio_snd_jack_info>,
|
||||
pcm_info: Vec<virtio_snd_pcm_info>,
|
||||
chmap_info: Vec<virtio_snd_chmap_info>,
|
||||
}
|
||||
|
||||
const SUPPORTED_FORMATS: u64 = 1 << VIRTIO_SND_PCM_FMT_U8
|
||||
| 1 << VIRTIO_SND_PCM_FMT_S16
|
||||
| 1 << VIRTIO_SND_PCM_FMT_S24
|
||||
| 1 << VIRTIO_SND_PCM_FMT_S32;
|
||||
const SUPPORTED_FRAME_RATES: u64 = 1 << VIRTIO_SND_PCM_RATE_8000
|
||||
| 1 << VIRTIO_SND_PCM_RATE_11025
|
||||
| 1 << VIRTIO_SND_PCM_RATE_16000
|
||||
| 1 << VIRTIO_SND_PCM_RATE_22050
|
||||
| 1 << VIRTIO_SND_PCM_RATE_44100
|
||||
| 1 << VIRTIO_SND_PCM_RATE_48000;
|
||||
|
||||
impl<'a> StreamInfo<'a> {
|
||||
async fn prepare(
|
||||
&mut self,
|
||||
ex: &Executor,
|
||||
mem: GuestMemory,
|
||||
tx_queue: &Rc<AsyncMutex<Queue>>,
|
||||
rx_queue: &Rc<AsyncMutex<Queue>>,
|
||||
interrupt: &Rc<Interrupt>,
|
||||
) -> Result<(), Error> {
|
||||
if self.state != VIRTIO_SND_R_PCM_SET_PARAMS
|
||||
&& self.state != VIRTIO_SND_R_PCM_PREPARE
|
||||
&& self.state != VIRTIO_SND_R_PCM_RELEASE
|
||||
{
|
||||
error!(
|
||||
"Invalid PCM state transition from {} to {}",
|
||||
get_virtio_snd_r_pcm_cmd_name(self.state),
|
||||
get_virtio_snd_r_pcm_cmd_name(VIRTIO_SND_R_PCM_PREPARE)
|
||||
);
|
||||
return Err(Error::OperationNotSupported);
|
||||
}
|
||||
let frame_size = self.channels as usize * self.format.sample_bytes();
|
||||
if self.period_bytes % frame_size != 0 {
|
||||
error!("period_bytes must be divisible by frame size");
|
||||
return Err(Error::OperationNotSupported);
|
||||
}
|
||||
if self.client.is_none() {
|
||||
// TODO(woodychow): once we're running in vm_concierge, we need an --enable-capture
|
||||
// option for
|
||||
// false: CrasClient::new()
|
||||
// true: CrasClient::with_type(CrasSocketType::Unified)
|
||||
// to use different socket.
|
||||
let mut client = CrasClient::new().map_err(Error::Libcras).unwrap();
|
||||
client.set_client_type(CrasClientType::CRAS_CLIENT_TYPE_CROSVM);
|
||||
self.client = Some(client);
|
||||
}
|
||||
// (*)
|
||||
// `buffer_size` in `audio_streams` API indicates the buffer size in bytes that the stream
|
||||
// consumes (or transmits) each time (next_playback/capture_buffer).
|
||||
// `period_bytes` in virtio-snd device (or ALSA) indicates the device transmits (or
|
||||
// consumes) for each PCM message.
|
||||
// Therefore, `buffer_size` in `audio_streams` == `period_bytes` in virtio-snd.
|
||||
let (stream, pcm_queue) = match self.direction {
|
||||
VIRTIO_SND_D_OUTPUT => (
|
||||
DirectionalStream::Output(
|
||||
self.client
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.new_async_playback_stream(
|
||||
self.channels as usize,
|
||||
self.format,
|
||||
self.frame_rate,
|
||||
// See (*)
|
||||
self.period_bytes / frame_size,
|
||||
&ex,
|
||||
)
|
||||
.map_err(Error::CreateStream)?
|
||||
.1,
|
||||
),
|
||||
tx_queue.clone(),
|
||||
),
|
||||
VIRTIO_SND_D_INPUT => {
|
||||
self.client.as_mut().unwrap().enable_cras_capture();
|
||||
(
|
||||
DirectionalStream::Input(
|
||||
self.client
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.new_async_capture_stream(
|
||||
self.channels as usize,
|
||||
self.format,
|
||||
self.frame_rate,
|
||||
// See (*)
|
||||
self.period_bytes / frame_size,
|
||||
&ex,
|
||||
)
|
||||
.map_err(Error::CreateStream)?
|
||||
.1,
|
||||
),
|
||||
rx_queue.clone(),
|
||||
)
|
||||
}
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let (sender, receiver) = mpsc::unbounded();
|
||||
self.sender = Some(sender);
|
||||
self.state = VIRTIO_SND_R_PCM_PREPARE;
|
||||
|
||||
self.status_mutex = Rc::new(AsyncMutex::new(WorkerStatus::Pause));
|
||||
self.cv = Rc::new(Condvar::new());
|
||||
let f = start_pcm_worker(
|
||||
ex.clone(),
|
||||
stream,
|
||||
receiver,
|
||||
self.status_mutex.clone(),
|
||||
self.cv.clone(),
|
||||
mem,
|
||||
pcm_queue,
|
||||
interrupt.clone(),
|
||||
self.period_bytes,
|
||||
);
|
||||
self.worker_future = Some(Box::new(ex.spawn_local(f).into_future()));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn start(&mut self) -> Result<(), Error> {
|
||||
if self.state != VIRTIO_SND_R_PCM_PREPARE && self.state != VIRTIO_SND_R_PCM_STOP {
|
||||
error!(
|
||||
"Invalid PCM state transition from {} to {}",
|
||||
get_virtio_snd_r_pcm_cmd_name(self.state),
|
||||
get_virtio_snd_r_pcm_cmd_name(VIRTIO_SND_R_PCM_START)
|
||||
);
|
||||
return Err(Error::OperationNotSupported);
|
||||
}
|
||||
self.state = VIRTIO_SND_R_PCM_START;
|
||||
*self.status_mutex.lock().await = WorkerStatus::Running;
|
||||
self.cv.notify_one();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn stop(&mut self) -> Result<(), Error> {
|
||||
if self.state != VIRTIO_SND_R_PCM_START {
|
||||
error!(
|
||||
"Invalid PCM state transition from {} to {}",
|
||||
get_virtio_snd_r_pcm_cmd_name(self.state),
|
||||
get_virtio_snd_r_pcm_cmd_name(VIRTIO_SND_R_PCM_STOP)
|
||||
);
|
||||
return Err(Error::OperationNotSupported);
|
||||
}
|
||||
self.state = VIRTIO_SND_R_PCM_STOP;
|
||||
*self.status_mutex.lock().await = WorkerStatus::Pause;
|
||||
self.cv.notify_one();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn release(&mut self) -> Result<(), Error> {
|
||||
if self.state != VIRTIO_SND_R_PCM_PREPARE && self.state != VIRTIO_SND_R_PCM_STOP {
|
||||
error!(
|
||||
"Invalid PCM state transition from {} to {}",
|
||||
get_virtio_snd_r_pcm_cmd_name(self.state),
|
||||
get_virtio_snd_r_pcm_cmd_name(VIRTIO_SND_R_PCM_RELEASE)
|
||||
);
|
||||
return Err(Error::OperationNotSupported);
|
||||
}
|
||||
self.state = VIRTIO_SND_R_PCM_RELEASE;
|
||||
self.release_worker().await?;
|
||||
self.client = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn release_worker(&mut self) -> Result<(), Error> {
|
||||
*self.status_mutex.lock().await = WorkerStatus::Quit;
|
||||
self.cv.notify_one();
|
||||
match self.sender.take() {
|
||||
Some(s) => s.close_channel(),
|
||||
None => (),
|
||||
}
|
||||
match self.worker_future.take() {
|
||||
Some(f) => f.await?,
|
||||
None => (),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct VirtioSndCras {
|
||||
cfg: virtio_snd_config,
|
||||
avail_features: u64,
|
||||
acked_features: u64,
|
||||
queue_sizes: Box<[u16]>,
|
||||
worker_threads: Vec<thread::JoinHandle<()>>,
|
||||
kill_evt: Option<Event>,
|
||||
}
|
||||
|
||||
impl VirtioSndCras {
|
||||
pub fn new(base_features: u64) -> Result<VirtioSndCras, Error> {
|
||||
let cfg = virtio_snd_config {
|
||||
jacks: 0.into(),
|
||||
streams: 2.into(),
|
||||
chmaps: 2.into(),
|
||||
};
|
||||
|
||||
let avail_features = base_features;
|
||||
|
||||
Ok(VirtioSndCras {
|
||||
cfg,
|
||||
avail_features,
|
||||
acked_features: 0,
|
||||
queue_sizes: vec![QUEUE_SIZE; NUM_QUEUES].into_boxed_slice(),
|
||||
worker_threads: Vec::new(),
|
||||
kill_evt: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl VirtioDevice for VirtioSndCras {
|
||||
fn keep_rds(&self) -> Vec<RawDescriptor> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn device_type(&self) -> u32 {
|
||||
TYPE_SOUND
|
||||
}
|
||||
|
||||
fn queue_max_sizes(&self) -> &[u16] {
|
||||
&self.queue_sizes
|
||||
}
|
||||
|
||||
fn features(&self) -> u64 {
|
||||
self.avail_features
|
||||
}
|
||||
|
||||
fn ack_features(&mut self, mut v: u64) {
|
||||
// Check if the guest is ACK'ing a feature that we didn't claim to have.
|
||||
let unrequested_features = v & !self.avail_features;
|
||||
if unrequested_features != 0 {
|
||||
warn!("virtio_fs got unknown feature ack: {:x}", v);
|
||||
|
||||
// Don't count these features as acked.
|
||||
v &= !unrequested_features;
|
||||
}
|
||||
self.acked_features |= v;
|
||||
}
|
||||
|
||||
fn read_config(&self, offset: u64, data: &mut [u8]) {
|
||||
copy_config(data, 0, self.cfg.as_slice(), offset)
|
||||
}
|
||||
|
||||
fn activate(
|
||||
&mut self,
|
||||
guest_mem: GuestMemory,
|
||||
interrupt: Interrupt,
|
||||
queues: Vec<Queue>,
|
||||
queue_evts: Vec<Event>,
|
||||
) {
|
||||
if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() {
|
||||
error!(
|
||||
"snd: expected {} queues, got {}",
|
||||
self.queue_sizes.len(),
|
||||
queues.len()
|
||||
);
|
||||
}
|
||||
|
||||
let (self_kill_evt, kill_evt) =
|
||||
match Event::new().and_then(|evt| Ok((evt.try_clone()?, evt))) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("failed to create kill Event pair: {}", e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.kill_evt = Some(self_kill_evt);
|
||||
|
||||
let mut jack_info: Vec<virtio_snd_jack_info> = Vec::new();
|
||||
let mut pcm_info: Vec<virtio_snd_pcm_info> = Vec::new();
|
||||
let mut chmap_info: Vec<virtio_snd_chmap_info> = Vec::new();
|
||||
|
||||
for i in 0..Into::<u32>::into(self.cfg.jacks) {
|
||||
let snd_info = virtio_snd_info {
|
||||
hda_fn_nid: i.into(),
|
||||
};
|
||||
// TODO(woodychow): Remove this hack
|
||||
// Assume this single device for now
|
||||
jack_info.push(virtio_snd_jack_info {
|
||||
hdr: snd_info,
|
||||
features: 0.into(),
|
||||
hda_reg_defconf: 0.into(),
|
||||
hda_reg_caps: 0.into(),
|
||||
connected: 0,
|
||||
padding: [0; 7],
|
||||
});
|
||||
}
|
||||
|
||||
// for _ in 0..(Into::<u32>::into(self.cfg.streams) as usize) {
|
||||
// TODO(woodychow): Remove this hack
|
||||
// Assume this single device for now
|
||||
pcm_info.push(virtio_snd_pcm_info {
|
||||
hdr: virtio_snd_info {
|
||||
hda_fn_nid: 0.into(),
|
||||
},
|
||||
features: 0.into(), /* 1 << VIRTIO_SND_PCM_F_XXX */
|
||||
formats: SUPPORTED_FORMATS.into(),
|
||||
rates: SUPPORTED_FRAME_RATES.into(),
|
||||
direction: VIRTIO_SND_D_OUTPUT,
|
||||
channels_min: 1,
|
||||
channels_max: 2,
|
||||
padding: [0; 5],
|
||||
});
|
||||
pcm_info.push(virtio_snd_pcm_info {
|
||||
hdr: virtio_snd_info {
|
||||
hda_fn_nid: 0.into(),
|
||||
},
|
||||
features: 0.into(), /* 1 << VIRTIO_SND_PCM_F_XXX */
|
||||
formats: SUPPORTED_FORMATS.into(),
|
||||
rates: SUPPORTED_FRAME_RATES.into(),
|
||||
direction: VIRTIO_SND_D_INPUT,
|
||||
channels_min: 1,
|
||||
channels_max: 2,
|
||||
padding: [0; 5],
|
||||
});
|
||||
// }
|
||||
|
||||
// for _ in 0..(Into::<u32>::into(self.cfg.chmaps) as usize) {
|
||||
|
||||
// Use stereo channel map.
|
||||
let mut positions = [VIRTIO_SND_CHMAP_NONE; VIRTIO_SND_CHMAP_MAX_SIZE];
|
||||
positions[0] = VIRTIO_SND_CHMAP_FL;
|
||||
positions[1] = VIRTIO_SND_CHMAP_FR;
|
||||
|
||||
chmap_info.push(virtio_snd_chmap_info {
|
||||
hdr: virtio_snd_info {
|
||||
hda_fn_nid: 0.into(),
|
||||
},
|
||||
direction: VIRTIO_SND_D_OUTPUT,
|
||||
channels: 2,
|
||||
positions,
|
||||
});
|
||||
chmap_info.push(virtio_snd_chmap_info {
|
||||
hdr: virtio_snd_info {
|
||||
hda_fn_nid: 0.into(),
|
||||
},
|
||||
direction: VIRTIO_SND_D_INPUT,
|
||||
channels: 2,
|
||||
positions,
|
||||
});
|
||||
// }
|
||||
|
||||
let worker_result = thread::Builder::new()
|
||||
.name("virtio_snd w".to_string())
|
||||
.spawn(move || {
|
||||
let mut streams: Vec<AsyncMutex<StreamInfo>> = Vec::new();
|
||||
streams.resize_with(pcm_info.len(), Default::default);
|
||||
|
||||
let streams = Rc::new(AsyncMutex::new(streams));
|
||||
|
||||
let snd_data = SndData {
|
||||
jack_info,
|
||||
pcm_info,
|
||||
chmap_info,
|
||||
};
|
||||
|
||||
if let Err(err_string) = run_worker(
|
||||
interrupt, queues, guest_mem, streams, snd_data, queue_evts, kill_evt,
|
||||
) {
|
||||
error!("{}", err_string);
|
||||
}
|
||||
});
|
||||
|
||||
match worker_result {
|
||||
Err(e) => {
|
||||
error!("failed to spawn virtio_snd worker: {}", e);
|
||||
return;
|
||||
}
|
||||
Ok(join_handle) => self.worker_threads.push(join_handle),
|
||||
}
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> bool {
|
||||
if let Some(kill_evt) = self.kill_evt.take() {
|
||||
// Ignore the result because there is nothing we can do about it.
|
||||
let _ = kill_evt.write(1);
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for VirtioSndCras {
|
||||
fn drop(&mut self) {
|
||||
self.reset();
|
||||
}
|
||||
}
|
||||
|
||||
fn run_worker(
|
||||
interrupt: Interrupt,
|
||||
mut queues: Vec<Queue>,
|
||||
mem: GuestMemory,
|
||||
streams: Rc<AsyncMutex<Vec<AsyncMutex<StreamInfo<'_>>>>>,
|
||||
snd_data: SndData,
|
||||
queue_evts: Vec<Event>,
|
||||
kill_evt: Event,
|
||||
) -> Result<(), String> {
|
||||
let ex = Executor::new().expect("Failed to create an executor");
|
||||
|
||||
let interrupt = Rc::new(interrupt);
|
||||
|
||||
let ctrl_queue = queues.remove(0);
|
||||
let _event_queue = queues.remove(0);
|
||||
let tx_queue = Rc::new(AsyncMutex::new(queues.remove(0)));
|
||||
let rx_queue = Rc::new(AsyncMutex::new(queues.remove(0)));
|
||||
|
||||
let mut evts_async: Vec<EventAsync> = queue_evts
|
||||
.into_iter()
|
||||
.map(|e| EventAsync::new(e.0, &ex).expect("Failed to create async event for queue"))
|
||||
.collect();
|
||||
|
||||
let ctrl_queue_evt = evts_async.remove(0);
|
||||
let _event_queue_evt = evts_async.remove(0);
|
||||
let tx_queue_evt = evts_async.remove(0);
|
||||
let rx_queue_evt = evts_async.remove(0);
|
||||
|
||||
let f_ctrl = handle_ctrl_queue(
|
||||
&ex,
|
||||
&mem,
|
||||
&streams,
|
||||
&snd_data,
|
||||
ctrl_queue,
|
||||
ctrl_queue_evt,
|
||||
&interrupt,
|
||||
&tx_queue,
|
||||
&rx_queue,
|
||||
);
|
||||
pin_mut!(f_ctrl);
|
||||
|
||||
// TODO(woodychow): Enable this when libcras sends jack connect/disconnect evts
|
||||
// let f_event = handle_event_queue(
|
||||
// &mem,
|
||||
// snd_state,
|
||||
// event_queue,
|
||||
// event_queue_evt,
|
||||
// interrupt,
|
||||
// );
|
||||
// pin_mut!(f_event);
|
||||
|
||||
let f_tx = handle_pcm_queue(&mem, &streams, &tx_queue, tx_queue_evt, &interrupt);
|
||||
pin_mut!(f_tx);
|
||||
|
||||
let f_rx = handle_pcm_queue(&mem, &streams, &rx_queue, rx_queue_evt, &interrupt);
|
||||
pin_mut!(f_rx);
|
||||
|
||||
// Exit if the kill event is triggered.
|
||||
let kill_evt = EventAsync::new(kill_evt.0, &ex).expect("failed to set up the kill event");
|
||||
let f_kill = wait_kill(kill_evt);
|
||||
pin_mut!(f_kill);
|
||||
|
||||
match ex.run_until(select4(f_ctrl, f_tx, f_rx, f_kill)) {
|
||||
Ok((ctrl_res, tx_res, rx_res, _kill_res)) => {
|
||||
if let SelectResult::Finished(Err(e)) = ctrl_res {
|
||||
return Err(format!("Error in handling ctrl queue: {}", e));
|
||||
}
|
||||
if let SelectResult::Finished(Err(e)) = tx_res {
|
||||
return Err(format!("Error in handling tx queue: {}", e));
|
||||
}
|
||||
if let SelectResult::Finished(Err(e)) = rx_res {
|
||||
return Err(format!("Error in handling rx queue: {}", e));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Error happened in executor: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
|
@ -4,9 +4,11 @@
|
|||
|
||||
pub mod common;
|
||||
pub mod constants;
|
||||
|
||||
pub mod layout;
|
||||
|
||||
#[cfg(feature = "audio_cras")]
|
||||
pub mod cras_backend;
|
||||
|
||||
pub mod vios_backend;
|
||||
|
||||
pub use vios_backend::new_sound;
|
||||
|
|
11
seccomp/aarch64/cras_snd_device.policy
Normal file
11
seccomp/aarch64/cras_snd_device.policy
Normal file
|
@ -0,0 +1,11 @@
|
|||
# Copyright 2021 The Chromium OS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
@include /usr/share/policy/crosvm/common_device.policy
|
||||
|
||||
openat: return ENOENT
|
||||
socket: arg0 == AF_UNIX
|
||||
socketpair: arg0 == AF_UNIX
|
||||
prctl: arg0 == PR_SET_NAME
|
||||
connect: 1
|
12
seccomp/arm/cras_snd_device.policy
Normal file
12
seccomp/arm/cras_snd_device.policy
Normal file
|
@ -0,0 +1,12 @@
|
|||
# Copyright 2021 The Chromium OS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
@include /usr/share/policy/crosvm/common_device.policy
|
||||
|
||||
open: return ENOENT
|
||||
openat: return ENOENT
|
||||
socket: arg0 == AF_UNIX
|
||||
socketpair: arg0 == AF_UNIX
|
||||
prctl: arg0 == PR_SET_NAME
|
||||
connect: 1
|
12
seccomp/x86_64/cras_snd_device.policy
Normal file
12
seccomp/x86_64/cras_snd_device.policy
Normal file
|
@ -0,0 +1,12 @@
|
|||
# Copyright 2021 The Chromium OS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
@include /usr/share/policy/crosvm/common_device.policy
|
||||
|
||||
open: return ENOENT
|
||||
openat: return ENOENT
|
||||
socket: arg0 == AF_UNIX
|
||||
socketpair: arg0 == AF_UNIX
|
||||
prctl: arg0 == PR_SET_NAME
|
||||
connect: 1
|
|
@ -209,6 +209,8 @@ pub struct Config {
|
|||
pub vcpu_affinity: Option<VcpuAffinity>,
|
||||
pub cpu_clusters: Vec<Vec<usize>>,
|
||||
pub cpu_capacity: BTreeMap<usize, u32>, // CPU index -> capacity
|
||||
#[cfg(feature = "audio_cras")]
|
||||
pub cras_snd: bool,
|
||||
pub delay_rt: bool,
|
||||
pub no_smt: bool,
|
||||
pub memory: Option<u64>,
|
||||
|
@ -294,6 +296,8 @@ impl Default for Config {
|
|||
vcpu_affinity: None,
|
||||
cpu_clusters: Vec::new(),
|
||||
cpu_capacity: BTreeMap::new(),
|
||||
#[cfg(feature = "audio_cras")]
|
||||
cras_snd: false,
|
||||
delay_rt: false,
|
||||
no_smt: false,
|
||||
memory: None,
|
||||
|
|
|
@ -45,6 +45,8 @@ pub enum Error {
|
|||
ConfigureHotPlugDevice(<Arch as LinuxArch>::Error),
|
||||
ConfigureVcpu(<Arch as LinuxArch>::Error),
|
||||
ConnectTube(io::Error),
|
||||
#[cfg(feature = "audio_cras")]
|
||||
CrasSoundDeviceNew(virtio::snd::cras_backend::Error),
|
||||
#[cfg(feature = "audio")]
|
||||
CreateAc97(devices::PciDeviceError),
|
||||
CreateConsole(devices::SerialError),
|
||||
|
@ -176,6 +178,8 @@ impl Display for Error {
|
|||
ConfigureHotPlugDevice(e) => write!(f, "Failed to configure pci hotplug device:{}", e),
|
||||
ConfigureVcpu(e) => write!(f, "failed to configure vcpu: {}", e),
|
||||
ConnectTube(e) => write!(f, "failed to connect to tube: {}", e),
|
||||
#[cfg(feature = "audio_cras")]
|
||||
CrasSoundDeviceNew(e) => write!(f, "failed to create cras sound device: {}", e),
|
||||
#[cfg(feature = "audio")]
|
||||
CreateAc97(e) => write!(f, "failed to create ac97 device: {}", e),
|
||||
CreateConsole(e) => write!(f, "failed to create console device: {}", e),
|
||||
|
|
41
src/linux.rs
41
src/linux.rs
|
@ -331,6 +331,40 @@ fn create_rng_device(cfg: &Config) -> DeviceResult {
|
|||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "audio_cras")]
|
||||
fn create_cras_snd_device(cfg: &Config) -> DeviceResult {
|
||||
let dev =
|
||||
virtio::snd::cras_backend::VirtioSndCras::new(virtio::base_features(cfg.protected_vm))
|
||||
.map_err(Error::CrasSoundDeviceNew)?;
|
||||
|
||||
let jail = match simple_jail(&cfg, "cras_snd_device")? {
|
||||
Some(mut jail) => {
|
||||
// Create a tmpfs in the device's root directory for cras_snd_device.
|
||||
// The size is 20*1024, or 20 KB.
|
||||
jail.mount_with_data(
|
||||
Path::new("none"),
|
||||
Path::new("/"),
|
||||
"tmpfs",
|
||||
(libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC) as usize,
|
||||
"size=20480",
|
||||
)?;
|
||||
|
||||
let run_cras_path = Path::new("/run/cras");
|
||||
jail.mount_bind(run_cras_path, run_cras_path, true)?;
|
||||
|
||||
add_current_user_to_jail(&mut jail)?;
|
||||
|
||||
Some(jail)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
Ok(VirtioDeviceStub {
|
||||
dev: Box::new(dev),
|
||||
jail,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "tpm")]
|
||||
fn create_tpm_device(cfg: &Config) -> DeviceResult {
|
||||
use std::ffi::CString;
|
||||
|
@ -1214,6 +1248,13 @@ fn create_virtio_devices(
|
|||
|
||||
devs.push(create_rng_device(cfg)?);
|
||||
|
||||
#[cfg(feature = "audio_cras")]
|
||||
{
|
||||
if cfg.cras_snd {
|
||||
devs.push(create_cras_snd_device(cfg)?);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tpm")]
|
||||
{
|
||||
if cfg.software_tpm {
|
||||
|
|
|
@ -964,6 +964,10 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
|
|||
"cpu-capacity" => {
|
||||
parse_cpu_capacity(value.unwrap(), &mut cfg.cpu_capacity)?;
|
||||
}
|
||||
#[cfg(feature = "audio_cras")]
|
||||
"cras-snd" => {
|
||||
cfg.cras_snd = true;
|
||||
}
|
||||
"no-smt" => {
|
||||
cfg.no_smt = true;
|
||||
}
|
||||
|
@ -2035,6 +2039,8 @@ fn run_vm(args: std::env::Args) -> std::result::Result<(), ()> {
|
|||
or colon-separated list of assignments of guest to host CPU assignments (e.g. 0=0:1=1:2=2) (default: no mask)"),
|
||||
Argument::value("cpu-cluster", "CPUSET", "Group the given CPUs into a cluster (default: no clusters)"),
|
||||
Argument::value("cpu-capacity", "CPU=CAP[,CPU=CAP[,...]]", "Set the relative capacity of the given CPU (default: no capacity)"),
|
||||
#[cfg(feature = "audio_cras")]
|
||||
Argument::flag("cras-snd", "Enable virtio-snd device with CRAS backend"),
|
||||
Argument::flag("no-smt", "Don't use SMT in the guest"),
|
||||
Argument::value("rt-cpus", "CPUSET", "Comma-separated list of CPUs or CPU ranges to run VCPUs on. (e.g. 0,1-3,5) (default: none)"),
|
||||
Argument::flag("delay-rt", "Don't set VCPUs real-time until make-rt command is run"),
|
||||
|
|
Loading…
Reference in a new issue