From 9e220e5e6d94acafa80e3fbb3d68cbc20e7f1aa5 Mon Sep 17 00:00:00 2001 From: Niculae Radu Date: Sat, 9 Apr 2022 09:54:33 +0300 Subject: [PATCH] Init virtio console. Added a virtio console device to replace the UART serial console. The implementation is divided between vm-virtio and vmm-reference. Signed-off-by: Niculae Radu --- Cargo.lock | 22 +++- src/api/src/lib.rs | 59 ++++++++- src/arch/Cargo.toml | 2 +- src/devices/Cargo.toml | 11 +- src/devices/src/virtio/console/device.rs | 122 ++++++++++++++++++ .../src/virtio/console/inorder_handler.rs | 114 ++++++++++++++++ src/devices/src/virtio/console/mod.rs | 23 ++++ .../src/virtio/console/queue_handler.rs | 117 +++++++++++++++++ src/devices/src/virtio/mod.rs | 1 + src/vm-vcpu-ref/Cargo.toml | 4 +- src/vm-vcpu/Cargo.toml | 4 +- src/vmm/Cargo.toml | 2 +- src/vmm/src/config/builder.rs | 69 +++++++++- src/vmm/src/config/mod.rs | 97 ++++++++++++++ src/vmm/src/lib.rs | 47 ++++++- src/vmm/tests/integration_tests.rs | 1 + tests/test_run_reference_vmm.py | 41 +++++- 17 files changed, 707 insertions(+), 29 deletions(-) create mode 100644 src/devices/src/virtio/console/device.rs create mode 100644 src/devices/src/virtio/console/inorder_handler.rs create mode 100644 src/devices/src/virtio/console/mod.rs create mode 100644 src/devices/src/virtio/console/queue_handler.rs diff --git a/Cargo.lock b/Cargo.lock index d32120e1..be408c02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,7 @@ dependencies = [ "log", "utils", "virtio-blk", + "virtio-console", "virtio-device", "virtio-queue", "vm-device", @@ -236,7 +237,7 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "virtio-blk" version = "0.1.0" -source = "git+https://github.com/rust-vmm/vm-virtio.git#d8ef45f57b46baa99e80e555deffd3fa1ab9affc" +source = "git+https://github.com/rust-vmm/vm-virtio.git?rev=1cde0af5c80aead5e44018029b5ecc0be83dea50#1cde0af5c80aead5e44018029b5ecc0be83dea50" dependencies = [ "log", "virtio-device", @@ -245,10 +246,19 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "virtio-console" +version = "0.1.0" +source = "git+https://github.com/rust-vmm/vm-virtio.git?rev=1cde0af5c80aead5e44018029b5ecc0be83dea50#1cde0af5c80aead5e44018029b5ecc0be83dea50" +dependencies = [ + "virtio-queue", + "vm-memory", +] + [[package]] name = "virtio-device" version = "0.1.0" -source = "git+https://github.com/rust-vmm/vm-virtio.git#d8ef45f57b46baa99e80e555deffd3fa1ab9affc" +source = "git+https://github.com/rust-vmm/vm-virtio.git?rev=1cde0af5c80aead5e44018029b5ecc0be83dea50#1cde0af5c80aead5e44018029b5ecc0be83dea50" dependencies = [ "log", "virtio-queue", @@ -257,8 +267,8 @@ dependencies = [ [[package]] name = "virtio-queue" -version = "0.2.0" -source = "git+https://github.com/rust-vmm/vm-virtio.git#d8ef45f57b46baa99e80e555deffd3fa1ab9affc" +version = "0.3.0" +source = "git+https://github.com/rust-vmm/vm-virtio.git?rev=1cde0af5c80aead5e44018029b5ecc0be83dea50#1cde0af5c80aead5e44018029b5ecc0be83dea50" dependencies = [ "log", "vm-memory", @@ -279,9 +289,9 @@ checksum = "f43fb5a6bd1a7d423ad72802801036719b7546cf847a103f8fe4575f5b0d45a6" [[package]] name = "vm-memory" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339d4349c126fdcd87e034631d7274370cf19eb0e87b33166bcd956589fc72c5" +checksum = "767ed8aaebbff902e02e6d3749dc2baef55e46565f8a6414a065e5baee4b4a81" dependencies = [ "libc", "winapi", diff --git a/src/api/src/lib.rs b/src/api/src/lib.rs index 9d23c992..6a43f3c0 100644 --- a/src/api/src/lib.rs +++ b/src/api/src/lib.rs @@ -50,6 +50,18 @@ impl Cli { .required(false) .takes_value(true) .help("Block device configuration. \n\tFormat: \"path=\"") + ) + .arg( + Arg::with_name("console") + .long("console") + .required(false) + .takes_value(true) + .help("Console configuration. \n\tFormat: \"type=\" \ + \nPossible values for \"type\":\ + \n\t * uart - Use the serial UART console.\ + \n\t * virtio - Use the virtio console.\ + \nUses the UART console by default if the \"--console\" \ + option is not present.") ); // Save the usage beforehand as a string, because `get_matches` consumes the `App`. @@ -69,6 +81,7 @@ impl Cli { .vcpu_config(matches.value_of("vcpu")) .net_config(matches.value_of("net")) .block_config(matches.value_of("block")) + .console_config(matches.value_of("console")) .build() .map_err(|e| format!("{:?}", e)) } @@ -82,7 +95,10 @@ mod tests { use linux_loader::cmdline::Cmdline; - use vmm::{KernelConfig, MemoryConfig, VcpuConfig, DEFAULT_KERNEL_LOAD_ADDR}; + use vmm::{ + ConsoleConfig, ConsoleType, KernelConfig, MemoryConfig, VcpuConfig, + DEFAULT_KERNEL_LOAD_ADDR, + }; #[test] fn test_launch() { @@ -214,7 +230,40 @@ mod tests { let mut foo_cmdline = Cmdline::new(4096); foo_cmdline.insert_str("\"foo=bar bar=foo\"").unwrap(); - // OK. + // OK. Virtio console. + assert_eq!( + Cli::launch(vec![ + "foobar", + "--memory", + "size_mib=128", + "--vcpu", + "num=1", + "--kernel", + "path=/foo/bar,cmdline=\"foo=bar bar=foo\",kernel_load_addr=42", + "--console", + "type=virtio", + ]) + .unwrap(), + VMMConfig { + kernel_config: KernelConfig { + path: PathBuf::from("/foo/bar"), + cmdline: foo_cmdline, + load_addr: 42, + }, + memory_config: MemoryConfig { size_mib: 128 }, + vcpu_config: VcpuConfig { num: 1 }, + block_config: None, + net_config: None, + console_config: Some(ConsoleConfig { + console_type: ConsoleType::Virtio + }), + } + ); + + let mut foo_cmdline = Cmdline::new(4096); + foo_cmdline.insert_str("\"foo=bar bar=foo\"").unwrap(); + + // OK. UART console. assert_eq!( Cli::launch(vec![ "foobar", @@ -224,6 +273,8 @@ mod tests { "num=1", "--kernel", "path=/foo/bar,cmdline=\"foo=bar bar=foo\",kernel_load_addr=42", + "--console", + "type=uart", ]) .unwrap(), VMMConfig { @@ -236,6 +287,9 @@ mod tests { vcpu_config: VcpuConfig { num: 1 }, block_config: None, net_config: None, + console_config: Some(ConsoleConfig { + console_type: ConsoleType::Uart + }), } ); @@ -252,6 +306,7 @@ mod tests { vcpu_config: VcpuConfig { num: 1 }, block_config: None, net_config: None, + console_config: None } ); } diff --git a/src/arch/Cargo.toml b/src/arch/Cargo.toml index f3a2eb9b..94d4fb26 100644 --- a/src/arch/Cargo.toml +++ b/src/arch/Cargo.toml @@ -8,4 +8,4 @@ edition = "2018" [dependencies] vm-fdt = "0.2.0" -vm-memory = "0.7.0" +vm-memory = "0.8.0" diff --git a/src/devices/Cargo.toml b/src/devices/Cargo.toml index 423cd3fb..4bf75523 100644 --- a/src/devices/Cargo.toml +++ b/src/devices/Cargo.toml @@ -11,16 +11,17 @@ kvm-ioctls = "0.11.0" libc = "0.2.76" linux-loader = "0.4.0" log = "0.4.6" -vm-memory = "0.7.0" +vm-memory = "0.8.0" vm-superio = "0.5.0" vmm-sys-util = "0.8.0" vm-device = "0.1.0" -virtio-blk = { git = "https://github.com/rust-vmm/vm-virtio.git", features = ["backend-stdio"] } -virtio-device = { git = "https://github.com/rust-vmm/vm-virtio.git"} -virtio-queue = { git = "https://github.com/rust-vmm/vm-virtio.git"} +virtio-blk = { git = "https://github.com/rust-vmm/vm-virtio.git", rev = "1cde0af5c80aead5e44018029b5ecc0be83dea50", features = ["backend-stdio"] } +virtio-device = { git = "https://github.com/rust-vmm/vm-virtio.git", rev = "1cde0af5c80aead5e44018029b5ecc0be83dea50"} +virtio-queue = { git = "https://github.com/rust-vmm/vm-virtio.git", rev = "1cde0af5c80aead5e44018029b5ecc0be83dea50"} +virtio-console = { git = "https://github.com/rust-vmm/vm-virtio.git", rev = "1cde0af5c80aead5e44018029b5ecc0be83dea50"} utils = { path = "../utils" } [dev-dependencies] -vm-memory = { version = "0.7.0", features = ["backend-mmap"] } +vm-memory = { version = "0.8.0", features = ["backend-mmap"] } diff --git a/src/devices/src/virtio/console/device.rs b/src/devices/src/virtio/console/device.rs new file mode 100644 index 00000000..443e131d --- /dev/null +++ b/src/devices/src/virtio/console/device.rs @@ -0,0 +1,122 @@ +// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +use crate::virtio::console::CONSOLE_DEVICE_ID; +use crate::virtio::features::{VIRTIO_F_IN_ORDER, VIRTIO_F_RING_EVENT_IDX, VIRTIO_F_VERSION_1}; + +use std::borrow::{Borrow, BorrowMut}; +use std::io::stdout; +use std::ops::DerefMut; +use std::sync::{Arc, Mutex}; +use virtio_console::console; + +use super::inorder_handler::InOrderQueueHandler; +use crate::virtio::console::queue_handler::QueueHandler; +use crate::virtio::{CommonConfig, Env, SingleFdSignalQueue, QUEUE_MAX_SIZE}; +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; +use virtio_queue::Queue; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::MmioManager; +use vm_device::{DeviceMmio, MutDeviceMmio}; +use vm_memory::GuestAddressSpace; + +use super::{Error, Result}; + +pub struct Console { + cfg: CommonConfig, +} + +impl Console +where + M: GuestAddressSpace + Clone + Send + 'static, +{ + pub fn new(env: &mut Env) -> Result>> + where + // We're using this (more convoluted) bound so we can pass both references and smart + // pointers such as mutex guards here. + B: DerefMut, + B::Target: MmioManager>, + { + let device_features = + (1 << VIRTIO_F_VERSION_1) | (1 << VIRTIO_F_IN_ORDER) | (1 << VIRTIO_F_RING_EVENT_IDX); + let queues = vec![ + Queue::new(env.mem.clone(), QUEUE_MAX_SIZE), + Queue::new(env.mem.clone(), QUEUE_MAX_SIZE), + ]; + // TODO: Add a config space to implement the optional features of the console. + // For basic operation it can be left empty. + let config_space = Vec::new(); + let virtio_cfg = VirtioConfig::new(device_features, queues, config_space); + let common_cfg = CommonConfig::new(virtio_cfg, env).map_err(Error::Virtio)?; + let console = Arc::new(Mutex::new(Console { cfg: common_cfg })); + + env.register_mmio_device(console.clone()) + .map_err(Error::Virtio)?; + + Ok(console) + } +} + +impl VirtioDeviceType for Console { + fn device_type(&self) -> u32 { + CONSOLE_DEVICE_ID + } +} + +impl Borrow> for Console { + fn borrow(&self) -> &VirtioConfig { + &self.cfg.virtio + } +} + +impl BorrowMut> for Console { + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.cfg.virtio + } +} + +impl VirtioDeviceActions for Console { + type E = Error; + + fn activate(&mut self) -> Result<()> { + let driver_notify = SingleFdSignalQueue { + irqfd: self.cfg.irqfd.clone(), + interrupt_status: self.cfg.virtio.interrupt_status.clone(), + }; + + let mut ioevents = self.cfg.prepare_activate().map_err(Error::Virtio)?; + + let inner = InOrderQueueHandler { + driver_notify, + receiveq: self.cfg.virtio.queues.remove(0), + transmitq: self.cfg.virtio.queues.remove(0), + console: console::Console::new_with_capacity(console::DEFAULT_CAPACITY, stdout()) + .map_err(Error::Console)?, + }; + + let handler = Arc::new(Mutex::new(QueueHandler { + inner, + receiveqfd: ioevents.remove(0), + transmitqfd: ioevents.remove(0), + })); + + self.cfg.finalize_activate(handler).map_err(Error::Virtio) + } + + fn reset(&mut self) -> Result<()> { + // Not implemented for now. + Ok(()) + } +} + +impl VirtioMmioDevice for Console {} + +impl MutDeviceMmio for Console { + fn mmio_read(&mut self, _base: MmioAddress, offset: u64, data: &mut [u8]) { + self.read(offset, data); + } + + fn mmio_write(&mut self, _base: MmioAddress, offset: u64, data: &[u8]) { + self.write(offset, data); + } +} diff --git a/src/devices/src/virtio/console/inorder_handler.rs b/src/devices/src/virtio/console/inorder_handler.rs new file mode 100644 index 00000000..2ecfd9c3 --- /dev/null +++ b/src/devices/src/virtio/console/inorder_handler.rs @@ -0,0 +1,114 @@ +// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +use crate::virtio::console::{RECEIVEQ_INDEX, TRANSMITQ_INDEX}; +use crate::virtio::SignalUsedQueue; +use std::io::Write; +use std::result; +use virtio_console::console; +use virtio_console::console::Error::UnexpectedReadOnlyDescriptor; +use virtio_queue::{Queue, QueueStateOwnedT, QueueStateT}; +use vm_memory::GuestAddressSpace; + +#[derive(Debug)] +pub enum Error { + GuestMemory(vm_memory::GuestMemoryError), + Queue(virtio_queue::Error), + Console(console::Error), +} + +impl From for Error { + fn from(e: vm_memory::GuestMemoryError) -> Self { + Error::GuestMemory(e) + } +} + +impl From for Error { + fn from(e: virtio_queue::Error) -> Self { + Error::Queue(e) + } +} + +impl From for Error { + fn from(e: console::Error) -> Self { + Error::Console(e) + } +} + +pub struct InOrderQueueHandler { + pub driver_notify: S, + pub transmitq: Queue, + pub receiveq: Queue, + pub console: console::Console, +} + +impl InOrderQueueHandler +where + M: GuestAddressSpace, + S: SignalUsedQueue, + T: Write, +{ + pub fn process_transmitq(&mut self) -> result::Result<(), Error> { + // This is done in a loop to catch the notifications that might be available when they are + // enabled again. More details can be found at `Queue::enable_notification`. + loop { + self.transmitq.disable_notification()?; + + while let Some(mut chain) = self + .transmitq + .state + .pop_descriptor_chain(self.transmitq.mem.memory()) + { + self.console.process_transmitq_chain(&mut chain)?; + + self.transmitq.add_used(chain.head_index(), 0)?; + + if self.transmitq.needs_notification()? { + self.driver_notify.signal_used_queue(TRANSMITQ_INDEX); + } + } + if !self.transmitq.enable_notification()? { + break; + } + } + Ok(()) + } + + pub fn process_receiveq(&mut self) -> result::Result<(), Error> { + // This is done in a loop to catch the notifications that might be available when they are + // enabled again. More details can be found at `Queue::enable_notification`. + loop { + self.receiveq.disable_notification()?; + while let Some(mut chain) = self + .receiveq + .state + .pop_descriptor_chain(self.receiveq.mem.memory()) + { + let used_len = match self.console.process_receiveq_chain(&mut chain) { + Ok(used_len) => { + if used_len == 0 { + self.receiveq.state.go_to_previous_position(); + break; + } + used_len + } + Err(UnexpectedReadOnlyDescriptor) => 0, + Err(e) => { + self.receiveq.state.go_to_previous_position(); + return Err(Error::Console(e)); + } + }; + + self.receiveq.add_used(chain.head_index(), used_len)?; + + if self.receiveq.needs_notification()? { + self.driver_notify.signal_used_queue(RECEIVEQ_INDEX); + } + } + if self.console.is_input_buffer_empty() || !self.receiveq.enable_notification()? { + break; + } + } + Ok(()) + } +} diff --git a/src/devices/src/virtio/console/mod.rs b/src/devices/src/virtio/console/mod.rs new file mode 100644 index 00000000..855af368 --- /dev/null +++ b/src/devices/src/virtio/console/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +mod device; +mod inorder_handler; +mod queue_handler; + +pub use device::Console; + +// Console device ID as defined by the standard. +pub const CONSOLE_DEVICE_ID: u32 = 3; + +// Numbers that represent the order of the queues for the basic virtio console without multiport +// support. +const RECEIVEQ_INDEX: u16 = 0; +const TRANSMITQ_INDEX: u16 = 1; + +#[derive(Debug)] +pub enum Error { + Virtio(crate::virtio::Error), + Console(virtio_console::console::Error), +} +pub type Result = std::result::Result; diff --git a/src/devices/src/virtio/console/queue_handler.rs b/src/devices/src/virtio/console/queue_handler.rs new file mode 100644 index 00000000..66674925 --- /dev/null +++ b/src/devices/src/virtio/console/queue_handler.rs @@ -0,0 +1,117 @@ +// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +use crate::virtio::console::Error; +use event_manager::{EventOps, Events, MutEventSubscriber}; +use log::error; +use std::io::{stdin, Read, Stdout}; +use vm_memory::GuestAddressSpace; +use vmm_sys_util::epoll::EventSet; +use vmm_sys_util::eventfd::EventFd; + +use crate::virtio::console::inorder_handler::InOrderQueueHandler; +use crate::virtio::SingleFdSignalQueue; + +const STDIN_EVENT: u32 = 0; +const TRANSMITQ_EVENT: u32 = 1; +const RECEIVEQ_EVENT: u32 = 2; + +const STDIN_BUFFER_SIZE: usize = 1024; + +// This object simply combines the more generic `ConsoleHandler` with a concrete queue +// signalling implementation based on `EventFd`s, and then also implements `MutEventSubscriber` +// to interact with the event manager. `transmitqfd` and `receiveqfd` are connected to queue +// notifications coming from the driver. +pub(crate) struct QueueHandler { + pub inner: InOrderQueueHandler, + pub transmitqfd: EventFd, + pub receiveqfd: EventFd, +} + +impl MutEventSubscriber for QueueHandler { + fn process(&mut self, events: Events, ops: &mut EventOps) { + match events.data() { + STDIN_EVENT => { + let mut out = [0u8; STDIN_BUFFER_SIZE]; + loop { + match stdin().read(&mut out) { + Err(e) => { + error!("Error while reading stdin: {:?}", e); + break; + } + Ok(count) => { + let event_set = events.event_set(); + let unregister_condition = event_set.contains(EventSet::ERROR) + | event_set.contains(EventSet::HANG_UP); + if count > 0 { + // Send bytes if the `input_buffer` is full. + if self.inner.console.available_capacity() < count { + if let Err(e) = self.inner.process_receiveq() { + error!("Receiveq processing failed: {:?}", e); + } + } + self.inner + .console + .enqueue_data(&mut out[..count].to_vec()) + .map_err(Error::Console) + .unwrap(); + + // Send bytes if input sequence is over. + if count < STDIN_BUFFER_SIZE { + if let Err(e) = self.inner.process_receiveq() { + error!("Receiveq processing failed: {:?}", e); + } + break; + } + } else if unregister_condition { + // Got 0 bytes from serial input; is it a hang-up or error? + ops.remove(events) + .expect("Failed to unregister serial input"); + break; + } + } + } + } + } + TRANSMITQ_EVENT => { + if let Err(e) = self.transmitqfd.read() { + error!("Could not read transmitq event fd: {:?}", e); + } else if let Err(e) = self.inner.process_transmitq() { + error!("Transmitq processing failed: {:?}", e); + } + } + RECEIVEQ_EVENT => { + if let Err(e) = self.receiveqfd.read() { + error!("Could not read receiveq event fd: {:?}", e); + } else if let Err(e) = self.inner.process_receiveq() { + error!("Receiveq processing failed: {:?}", e); + } + } + _ => { + error!( + "Received unknown event data for virtio console: {}", + events.data() + ); + } + } + } + + fn init(&mut self, ops: &mut EventOps) { + ops.add(Events::with_data(&stdin(), STDIN_EVENT, EventSet::IN)) + .expect("Failed to register stdin event"); + + ops.add(Events::with_data( + &self.transmitqfd, + TRANSMITQ_EVENT, + EventSet::IN, + )) + .expect("Failed to register transmitq event"); + + ops.add(Events::with_data( + &self.receiveqfd, + RECEIVEQ_EVENT, + EventSet::IN, + )) + .expect("Failed to register receiveq event"); + } +} diff --git a/src/devices/src/virtio/mod.rs b/src/devices/src/virtio/mod.rs index 16a4b658..fbc73206 100644 --- a/src/devices/src/virtio/mod.rs +++ b/src/devices/src/virtio/mod.rs @@ -4,6 +4,7 @@ // We're only providing virtio over MMIO devices for now, but we aim to add PCI support as well. pub mod block; +pub mod console; pub mod net; use std::convert::TryFrom; diff --git a/src/vm-vcpu-ref/Cargo.toml b/src/vm-vcpu-ref/Cargo.toml index 2080f4af..ff1e94bc 100644 --- a/src/vm-vcpu-ref/Cargo.toml +++ b/src/vm-vcpu-ref/Cargo.toml @@ -13,9 +13,9 @@ keywords = ["virt", "kvm", "vm"] thiserror = "1.0.30" kvm-ioctls = { version = "0.11.0" } kvm-bindings = { version = "0.5.0", features = ["fam-wrappers"] } -vm-memory = "0.7.0" +vm-memory = "0.8.0" libc = "0.2.76" [dev-dependencies] -vm-memory = { version = "0.7.0", features = ["backend-mmap"] } +vm-memory = { version = "0.8.0", features = ["backend-mmap"] } vmm-sys-util = "0.8.0" diff --git a/src/vm-vcpu/Cargo.toml b/src/vm-vcpu/Cargo.toml index a3f878fb..79f0396f 100644 --- a/src/vm-vcpu/Cargo.toml +++ b/src/vm-vcpu/Cargo.toml @@ -11,7 +11,7 @@ thiserror = "1.0.30" libc = "0.2.76" kvm-bindings = { version = "0.5.0", features = ["fam-wrappers"] } kvm-ioctls = "0.11.0" -vm-memory = "0.7.0" +vm-memory = "0.8.0" vmm-sys-util = ">=0.8.0" vm-device = "0.1.0" @@ -20,4 +20,4 @@ vm-vcpu-ref = { path = "../vm-vcpu-ref" } arch = { path = "../arch" } [dev-dependencies] -vm-memory = { version = "0.7.0", features = ["backend-mmap"] } +vm-memory = { version = "0.8.0", features = ["backend-mmap"] } diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index da5d13fc..842dacaf 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -10,7 +10,7 @@ kvm-bindings = { version = "0.5.0", features = ["fam-wrappers"] } kvm-ioctls = "0.11.0" libc = "0.2.91" linux-loader = { version = "0.4.0", features = ["bzimage", "elf"] } -vm-memory = { version = "0.7.0", features = ["backend-mmap"] } +vm-memory = { version = "0.8.0", features = ["backend-mmap"] } vm-superio = "0.5.0" vmm-sys-util = "0.8.0" vm-device = "0.1.0" diff --git a/src/vmm/src/config/builder.rs b/src/vmm/src/config/builder.rs index 30fb0a25..09bb1eb0 100644 --- a/src/vmm/src/config/builder.rs +++ b/src/vmm/src/config/builder.rs @@ -5,7 +5,8 @@ use std::convert::TryFrom; use super::{ - BlockConfig, ConversionError, KernelConfig, MemoryConfig, NetConfig, VMMConfig, VcpuConfig, + BlockConfig, ConsoleConfig, ConversionError, KernelConfig, MemoryConfig, NetConfig, VMMConfig, + VcpuConfig, }; /// Builder structure for VMMConfig @@ -168,6 +169,26 @@ impl Builder { } } + /// Configure Builder with Console Configuration for the VMM. + /// + /// # Example + /// + /// You can see example of how to use this function in [`Example` section from + /// `build`](#method.build) + pub fn console_config(self, console: Option) -> Self + where + ConsoleConfig: TryFrom, + >::Error: Into, + { + match console { + Some(c) => self.and_then(|mut config| { + config.console_config = Some(TryFrom::try_from(c).map_err(Into::into)?); + Ok(config) + }), + None => self, + } + } + fn and_then(self, func: F) -> Self where F: FnOnce(VMMConfig) -> Result, @@ -184,7 +205,7 @@ mod tests { use std::path::PathBuf; use super::*; - use crate::DEFAULT_KERNEL_LOAD_ADDR; + use crate::{ConsoleType, DEFAULT_KERNEL_LOAD_ADDR}; #[test] fn test_builder_default_err() { @@ -313,6 +334,46 @@ mod tests { ); } + #[test] + fn test_builder_console_config_none_default() { + let vmm_config = Builder::default() + .console_config(None as Option<&str>) + .kernel_config(Some("path=bzImage")) + .build(); + assert!(vmm_config.is_ok()); + assert!(vmm_config.unwrap().console_config.is_none()); + } + + #[test] + fn test_builder_console_config_success_uart() { + let vmm_config = Builder::default() + .console_config(Some("type=uart")) + .kernel_config(Some("path=bzImage")) + .build(); + assert!(vmm_config.is_ok()); + assert_eq!( + vmm_config.unwrap().console_config, + Some(ConsoleConfig { + console_type: ConsoleType::Uart, + }) + ); + } + + #[test] + fn test_builder_console_config_success_virtio() { + let vmm_config = Builder::default() + .console_config(Some("type=virtio")) + .kernel_config(Some("path=bzImage")) + .build(); + assert!(vmm_config.is_ok()); + assert_eq!( + vmm_config.unwrap().console_config, + Some(ConsoleConfig { + console_type: ConsoleType::Virtio, + }) + ); + } + #[test] fn test_builder_vmm_config_success() { let vmm_config = Builder::default() @@ -321,6 +382,7 @@ mod tests { .net_config(Some("tap=tap0")) .kernel_config(Some("path=bzImage")) .block_config(Some("path=/dev/loop0")) + .console_config(Some("type=uart")) .build(); assert!(vmm_config.is_ok()); assert_eq!( @@ -338,6 +400,9 @@ mod tests { }), block_config: Some(BlockConfig { path: PathBuf::from("/dev/loop0") + }), + console_config: Some(ConsoleConfig { + console_type: ConsoleType::Uart, }) } ); diff --git a/src/vmm/src/config/mod.rs b/src/vmm/src/config/mod.rs index ba98af84..8918f497 100644 --- a/src/vmm/src/config/mod.rs +++ b/src/vmm/src/config/mod.rs @@ -32,6 +32,8 @@ pub enum ConversionError { ParseNet(String), /// Failed to parse the string representation for the block. ParseBlock(String), + /// Failed to parse the string representation for the console. + ParseConsole(String), } impl ConversionError { @@ -50,6 +52,9 @@ impl ConversionError { fn new_net(err: T) -> Self { Self::ParseNet(err.to_string()) } + fn new_console(err: T) -> Self { + Self::ParseConsole(err.to_string()) + } } impl VMMConfig { @@ -68,6 +73,7 @@ impl fmt::Display for ConversionError { ParseVcpus(ref s) => write!(f, "Invalid input for vCPUs: {}", s), ParseNet(ref s) => write!(f, "Invalid input for network: {}", s), ParseBlock(ref s) => write!(f, "Invalid input for block: {}", s), + ParseConsole(ref s) => write!(f, "Invalid input for console: {}", s), } } } @@ -258,6 +264,59 @@ impl TryFrom<&str> for BlockConfig { } } +/// Console type enum. +#[derive(Clone, Debug, PartialEq)] +pub enum ConsoleType { + /// Serial UART console. + Uart, + /// Virtio console. + Virtio, +} + +/// Console configuration. +#[derive(Clone, Debug, PartialEq)] +pub struct ConsoleConfig { + /// Type of console. + pub console_type: ConsoleType, +} + +impl Default for ConsoleConfig { + fn default() -> Self { + ConsoleConfig { + console_type: ConsoleType::Uart, + } + } +} + +impl TryFrom<&str> for ConsoleConfig { + type Error = ConversionError; + + fn try_from(console_cfg_str: &str) -> Result { + // Supported options: `type=String` + let mut arg_parser = CfgArgParser::new(console_cfg_str); + + let console_type_str: String = arg_parser + .value_of("type") + .map_err(ConversionError::new_console)? + .ok_or_else(|| ConversionError::new_console("Missing required argument: type"))?; + + let console_type = match console_type_str.as_str() { + "uart" => ConsoleType::Uart, + "virtio" => ConsoleType::Virtio, + arg => { + return Err(ConversionError::ParseConsole( + "Invalid argument for \"type\": ".to_string() + arg, + )) + } + }; + + arg_parser + .all_consumed() + .map_err(ConversionError::new_console)?; + Ok(ConsoleConfig { console_type }) + } +} + /// VMM configuration. #[derive(Clone, Debug, Default, PartialEq)] pub struct VMMConfig { @@ -271,6 +330,8 @@ pub struct VMMConfig { pub net_config: Option, /// Block device configuration. pub block_config: Option, + /// Console configuration. + pub console_config: Option, } #[cfg(test)] @@ -417,4 +478,40 @@ mod tests { let memory_str = "size_mib=12,blah=blah"; assert!(MemoryConfig::try_from(memory_str).is_err()); } + + #[test] + fn test_console_config() { + let type_str = "type=uart"; + let console_cfg = ConsoleConfig::try_from(type_str).unwrap(); + let expected_cfg = ConsoleConfig { + console_type: ConsoleType::Uart, + }; + assert_eq!(console_cfg, expected_cfg); + + let type_str = "type=virtio"; + let console_cfg = ConsoleConfig::try_from(type_str).unwrap(); + let expected_cfg = ConsoleConfig { + console_type: ConsoleType::Virtio, + }; + assert_eq!(console_cfg, expected_cfg); + + // Test case: empty string error. + assert!(ConsoleConfig::try_from("").is_err()); + + // Test case: empty type name error. + let type_str = "type="; + assert!(ConsoleConfig::try_from(type_str).is_err()); + + // Test case: invalid string. + let type_str = "type=blah"; + assert!(ConsoleConfig::try_from(type_str).is_err()); + + // Test case: invalid argument name. + let type_str = "blah=uart"; + assert!(ConsoleConfig::try_from(type_str).is_err()); + + // Test case: unused parameters + let type_str = "type=uart,blah=blah"; + assert!(ConsoleConfig::try_from(type_str).is_err()); + } } diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index e42ddd14..ed80f8b7 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -56,6 +56,7 @@ use vmm_sys_util::{epoll::EventSet, eventfd::EventFd, terminal::Terminal}; use boot::build_bootparams; pub use config::*; use devices::virtio::block::{self, BlockArgs}; +use devices::virtio::console; use devices::virtio::net::{self, NetArgs}; use devices::virtio::{Env, MmioConfig}; @@ -118,6 +119,8 @@ pub enum MemoryError { pub enum Error { /// Failed to create block device. Block(block::Error), + /// Failed to create console device + Console(console::Error), /// Failed to write boot parameters to guest memory. #[cfg(target_arch = "x86_64")] BootConfigure(configurator::Error), @@ -169,6 +172,7 @@ pub type Result = std::result::Result; type Block = block::Block>; type Net = net::Net>; +type Console = console::Console>; /// A live VMM. pub struct Vmm { @@ -185,6 +189,7 @@ pub struct Vmm { exit_handler: WrappedExitHandler, block_devices: Vec>>, net_devices: Vec>>, + console_devices: Vec>>, // TODO: fetch the vcpu number from the `vm` object. // TODO-continued: this is needed to make the arm POC work as we need to create the FDT // TODO-continued: after the other resources are created. @@ -285,11 +290,18 @@ impl TryFrom for Vmm { exit_handler: wrapped_exit_handler, block_devices: Vec::new(), net_devices: Vec::new(), + console_devices: Vec::new(), #[cfg(target_arch = "aarch64")] num_vcpus: config.vcpu_config.num as u64, }; - - vmm.add_serial_console()?; + if let Some(cfg) = config.console_config.as_ref() { + match cfg.console_type { + ConsoleType::Uart => vmm.add_serial_console()?, + ConsoleType::Virtio => vmm.add_virtio_console()?, + } + } else { + vmm.add_serial_console()?; + } #[cfg(target_arch = "x86_64")] vmm.add_i8042_device()?; #[cfg(target_arch = "aarch64")] @@ -586,6 +598,35 @@ impl Vmm { Ok(()) } + fn add_virtio_console(&mut self) -> Result<()> { + let mem = Arc::new(self.guest_memory.clone()); + + self.kernel_cfg + .cmdline + .insert_str("console=hvc0") + .map_err(Error::Cmdline)?; + + let range = MmioRange::new(MmioAddress(MMIO_GAP_START + 0x4000), 0x1000).unwrap(); + let mmio_cfg = MmioConfig { range, gsi: 7 }; + + let mut guard = self.device_mgr.lock().unwrap(); + + let mut env = Env { + mem, + vm_fd: self.vm.vm_fd(), + event_mgr: &mut self.event_mgr, + mmio_mgr: guard.deref_mut(), + mmio_cfg, + kernel_cmdline: &mut self.kernel_cfg.cmdline, + }; + + // We can also hold this somewhere if we need to keep the handle for later. + let console = Console::new(&mut env).map_err(Error::Console)?; + self.console_devices.push(console); + + Ok(()) + } + // Helper function that computes the kernel_load_addr needed by the // VcpuState when creating the Vcpus. #[cfg(target_arch = "x86_64")] @@ -720,6 +761,7 @@ mod tests { vcpu_config: VcpuConfig { num: NUM_VCPUS }, block_config: None, net_config: None, + console_config: None, } } @@ -759,6 +801,7 @@ mod tests { exit_handler, block_devices: Vec::new(), net_devices: Vec::new(), + console_devices: Vec::new(), #[cfg(target_arch = "aarch64")] num_vcpus: vmm_config.vcpu_config.num as u64, } diff --git a/src/vmm/tests/integration_tests.rs b/src/vmm/tests/integration_tests.rs index 7f357ed3..95b33be0 100644 --- a/src/vmm/tests/integration_tests.rs +++ b/src/vmm/tests/integration_tests.rs @@ -29,6 +29,7 @@ fn run_vmm(kernel_path: PathBuf) { vcpu_config: default_vcpu_config(), block_config: None, net_config: None, + console_config: None, }; let mut vmm = Vmm::try_from(vmm_config).unwrap(); diff --git a/tests/test_run_reference_vmm.py b/tests/test_run_reference_vmm.py index fd9a2e1a..4a2c7338 100644 --- a/tests/test_run_reference_vmm.py +++ b/tests/test_run_reference_vmm.py @@ -14,6 +14,9 @@ from tools.s3 import s3_download +import random +import string + # No. of seconds after which to give up for the test TEST_TIMEOUT = 30 @@ -86,9 +89,9 @@ def default_disk(): """ -def start_vmm_process(kernel_path, disk_path=None, num_vcpus=1, mem_size_mib=1024 ,default_cmdline=False): +def start_vmm_process(kernel_path, disk_path=None, num_vcpus=1, mem_size_mib=1024, console_type="uart", default_cmdline=False): # Kernel config - cmdline = "console=ttyS0 i8042.nokbd reboot=t panic=1 pci=off" + cmdline = "i8042.nokbd reboot=t panic=1 pci=off" kernel_load_addr = 1048576 @@ -98,6 +101,7 @@ def start_vmm_process(kernel_path, disk_path=None, num_vcpus=1, mem_size_mib=102 "target/release/vmm-reference", "--memory", "size_mib={}".format(mem_size_mib), "--vcpu", "num={}".format(num_vcpus), + "--console", "type={}".format(console_type), "--kernel" ] if default_cmdline: @@ -205,16 +209,18 @@ def run_cmd_inside_vm(cmd, vmm_process, prompt, timeout=5): then = time.time() giveup_after = timeout + total_data = b'' while True: try: data = os.read(vmm_process.stdout.fileno(), 4096) - output_lines = data.split(b'\r\n') - last = output_lines[-1].strip() - if prompt in last: + total_data += data + crt_lines = data.split(b'\r\n') + last = crt_lines[-1].strip() + if prompt == last[-len(prompt):]: # FIXME: WE get the prompt twice in the output at the end, # So removing it. No idea why twice? # First one is 'cmd' - return output_lines[1:-2] + return total_data.split(b'\r\n')[1:-2] except BlockingIOError as _: time.sleep(1) @@ -390,3 +396,26 @@ def test_reference_vmm_mem(kernel): expect_mem(vmm_process, expected_mem_mib) shutdown(vmm_process) + + +@pytest.mark.parametrize("kernel,disk", UBUNTU_KERNEL_DISK_PAIRS) +def test_reference_vmm_virtio_console_perf(kernel, disk): + """Start the reference VMM and test the virtio console.""" + + vmm_process, tmp_disk_path = start_vmm_process(kernel, disk_path=disk, console_type="virtio") + + prompt = 'root@ubuntu-rust-vmm:~#' + login_string = 'ubuntu-rust-vmm login:' + expect_string(vmm_process, login_string) + + cmd = 'root' + output = run_cmd_inside_vm(cmd.encode(), vmm_process, prompt.encode(), timeout=5) + output = b''.join(output).decode() + assert 'Welcome to Ubuntu 20.04 LTS' in output + + cmd = 'echo ' + 10000 * '*' + output = run_cmd_inside_vm(cmd.encode(), vmm_process, prompt.encode(), timeout=5) + output = b''.join(output).decode() + assert output == 10000 * '*' + + shutdown(vmm_process)