ARM Virtualization

1. VIRTIO
Yi-Man Ma & Chang-Jung Lin
2011 / 01 / 05
Thursday, January 5, 12 1

2. Agenda
Introduction
Architecture
VirtQueue & Vring
Five APIs
Adding a new Virtio Device / Driver
Thursday, January 5, 12 2

3. INTRODUCTION
Thursday, January 5, 12 3

4. ct ion
du
In tro
Create a shortcut to deal with IO Request
Coalesced IO Request by VirtQueue
Thursday, January 5, 12 4

5. ct ion
du
In tro
Device emulation in full virtualization
and paravirtualization environments
Guest operating system Guest operating system
Para-drivers
Traps Interfaces
Hypervisor Hypervisor
(Full virtualization) Device emulation (Full virtualization) Device emulation
Hardware Hardware
Thursday, January 5, 12 5

6. ARCHITECTURE
Thursday, January 5, 12 6

7. tu re
hit ec
A rc
GUEST1 GUEST2
Transport Transport
Other
QEMU QEMU
Processes
KVM
Host
Hardware
Thursday, January 5, 12 7

8. tu re
hit ec
A rc
Virtio Driver
Guest
Virtio PCI Controller
Vring Transport
Virtio PCI Controller
QEMU
Virtio Device
Thursday, January 5, 12 8

9. VIRTQUEUE & VRING
Thursday, January 5, 12 9

10. ue ue
Vi rtQ
Each Device have it’s own virtqueues.
Queue Requests
Thursday, January 5, 12 10

11. V ri ng
Each virtqueue have it’s own vring.
Memory Mapped between QEMU & Guest
Actual memory layout for queue
Thursday, January 5, 12 11

12. ue ng
ue ri
Vi rtQ V
Find virtqueue Virtio Driver
Guest
Virtio PCI Controller
Alloc Vring
Vring Transport
Virtio PCI Controller
Vring GPA QEMU
Virtio Device
Thursday, January 5, 12 12

13. ue ng
ue ri
Vi rtQ V
Virtio Driver
Virt queue Write Virt queue Read
Vring Vring
Virtio Device
Thursday, January 5, 12 13

14. ue ng
ue ri
Vi rtQ V
Virtio Driver
Virt queue Write Virt queue Read
Vring Vring
Virtio Device
Thursday, January 5, 12 14

15. ue ng
ue ri
Vi rtQ V
Virtio Driver
Add Buf
Virt queue Write Virt queue Read
Vring In Out Vring
data
Virtio Device
Thursday, January 5, 12 15

16. ue ng
ue ri
Vi rtQ V
Kick Virtio Driver
Virt queue Write Virt queue Read
Vring In Out Vring
data
Virtio Device
Thursday, January 5, 12 16

17. ue ng
ue ri
Vi rtQ V
Virtio Driver
Virt queue Write Virt queue Read
Vring In Out Vring
data
POP
Virtio Device
Thursday, January 5, 12 17

18. ue ng
ue ri
Vi rtQ V
Virtio Driver
Virt queue Write Virt queue Read
Vring In Out Vring
data
Push
Virtio Device
Thursday, January 5, 12 18

19. ue ng
ue ri
Vi rtQ V
Virtio Driver Get Buf
Virt queue Write Virt queue Read
data
Vring In Out Vring
Virtio Device
Thursday, January 5, 12 19

20. FIVE APIS
Thursday, January 5, 12 20

21. APIs
F ive
add_buf
kick
get_buf
disable_cb
enable_cb
Thursday, January 5, 12 21

22. Is et buf
AP g
F ive add &
add_buf
expose buffer to other end
get_buf
get the next used buffer
Thursday, January 5, 12 22

23. Is
AP ick
F ive k
kick
update after add_buf
notify QEMU to deal with the data
Thursday, January 5, 12 23

24. Is ative
AP el
F ive C Br
disable_cb
disable callbacks
not necessarily synchronous
unreliable and only useful as an optimization
enable_cb
restart callbacks after disable_cb
Thursday, January 5, 12 24

25. NEW DEVICE / DRIVER
Thursday, January 5, 12 25

26. river
D
New
include/linux/virtio_ids.h
include/linux/virtio.h
include/linux/virtio_[device].h
drivers/virtio/virtio_[device].c
drivers/virtio/virtio_pci.c
Thursday, January 5, 12 26

27. ev ice
D
New
hw/virtio-[device].c
hw/virtio-[device].h
hw/pci.h
hw/virtio_pci.c
Thursday, January 5, 12 27

28. tu re
hit ec
A rc
Virtio Driver
Guest
Virtio PCI Controller
Vring Transport
Virtio PCI Controller
QEMU
Virtio Device
Thursday, January 5, 12 28

29. ev iceeid
D devic Virtio PCI Controller
New add
Modify qemu-src/hw/pci.h
#define PCI_DEVICE_ID_VIRTIO_[device] 0x1004
Thursday, January 5, 12 29

30. icde r
ev le
Dal han Virtio PCI Controller
Newiniti
Modify qemu-src/hw/virtio-pci.c
static int virtio_[device]_init_pci(PCIDevice *pci_dev)
{
VirtIOPCIProxy *proxy =
DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
VirtIODevice *vdev;
vdev = virtio_[device]_init(&pci_dev->qdev);
if (!vdev) {
return -1;
}
virtio_init_pci(proxy, vdev);
return 0;
}
Thursday, January 5, 12 30

31. ev ce
idler
Dt han Virtio PCI Controller
New exi
Modify qemu-src/hw/virtio-pci.c
static int virtio_[device]_exit_pci (PCIDevice *pci_dev)
{
VirtIOPCIProxy *proxy =
DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
virtio_pci_stop_ioeventfd(proxy);
virtio_[device]_exit(proxy->vdev);
return virtio_exit_pci(pci_dev);
}
Thursday, January 5, 12 31

32. ieciefo
ev n
D evic
wdd d
Virtio PCI Controller
Nea
Modify qemu-src/hw/virtio-pci.c
static PCIDeviceInfo virtio_info[] = {
{
.qdev.name = "virtio-[device]-pci",
.qdev.alias = "virtio-[device]",
.init = virtio_[device]_init_pci,
exit
. = virtio_[device]_exit_pci,
device_id = PCI_DEVICE_ID_VIRTIO_[device],
.
.qdev.props = (Property[]) {
DEFINE_VIRTIO_COMMON_FEATURES(
VirtIOPCIProxy, host_features),
DEFINE_PROP_END_OF_LIST(),
,
}
...... (Skip)
Thursday, January 5, 12 32

33. tu re
hit ec
A rc
Virtio Driver
Guest
Virtio PCI Controller
Vring Transport
Virtio PCI Controller
QEMU
Virtio Device
Thursday, January 5, 12 33

34. ev ice
D Virtio Device
New
Add virtio-[device].c, virtio[device].h
Put them under the directory qemu-src/hw/
Thursday, January 5, 12 34

35. ivcce].h
ev i e
D-[de Virtio Device
ewirtio
N v
#ifndef _QEMU_VIRTIO_[device]_H
#include "virtio.h“
#define _QEMU_VIRTIO_[device]_H
#define VIRTIO_ID_[device] [num] //ID for your device
VirtIODevice *virtio_[device]_init(DeviceState *dev);
void virtio_[device]_exit(VirtIODevice *vdev);
#endif
Thursday, January 5, 12 35

36. ivcce].c
ev i e
D-[de Virtio Device
ewirtio
N v
#include <qemu-common.h>
#include "qemu-error.h"
#include "virtio-[device].h"
#include "virtio.h"
typedef struct VirtIO[device]
{
VirtIODevice vdev;
VirtQueue *vq[num];
DeviceState *qdev;
int status;
} VirtIO[device];
struct virtio_[device]_config {
int config;
};
Thursday, January 5, 12 36

37. ictere
evuc u
D str Virtio Device
ewirtio
N v
struct VirtIODevice
{
...........
uint32_t (*get_features)(VirtIODevice *vdev, uint32_t
requested_features);
uint32_t (*bad_features)(VirtIODevice *vdev);
void (*set_features)(VirtIODevice *vdev, uint32_t val);
void (*get_config)(VirtIODevice *vdev, uint8_t *config);
void (*set_config)(VirtIODevice *vdev, const uint8_t *config);
void (*reset)(VirtIODevice *vdev);
void (*set_status)(VirtIODevice *vdev, uint8_t val);
.......
};
Thursday, January 5, 12 37

38. iece nt
evl me
Dimp Virtio Device
eweed
N n
static VirtIO[device] *to_virtio_[device](VirtIODevice *vdev)
//Get device defined by yourself
static uint32_t virtio_[device]_ get_features (VirtIODevice *vdev,
uint32_t features);
static uint32_t virtio_[device]_ bad_features (VirtIODevice
*vdev);
static void virtio_[device]_ set_features (VirtIODevice *vdev,
uint32_t val);
static void virtio_[device]_ get_config (VirtIODevice *vdev,
uint8_t *config);
static void virtio_[device]_ set_config (VirtIODevice *vdev, const
uint8_t *config);
Thursday, January 5, 12 38

39. iece nt
evl me
Dimp Virtio Device
eweed
N n
static void virtio_[device]_ reset (VirtIODevice *vdev);
static void virtio_[device]_ set_status (VirtIODevice *vdev,
uint8_t val);
VirtIODevice *virtio_[device]_init(DeviceState *dev);
//Initialize your device
void virtio_[device]_exit (VirtIODevice *vdev);
static void virtio_[device]_save(QEMUFile *f, void *opaque);
static int virtio_[device]_load(QEMUFile *f, void *opaque, int
version_id);
static void virtio_[device]_handle_out(VirtIODevice *vdev,
VirtQueue *vq);
Thursday, January 5, 12 39

40. itcees
ev ur
Dt fea Virtio Device
ew bou
N a
static uint32_t virtio_[device]_ get_features (VirtIODevice *vdev,
uint32_t features)
{
VirtIO[device] *n = to_virtio_[device](vdev);
return features;
}
static uint32_t virtio_[device]_ bad_features (VirtIODevice
*vdev){
uint32_t features = 0;
return features;
}
static void virtio_[device]_ set_features (VirtIODevice *vdev,
uint32_t val){
VirtIOMy *n = to_virtio_[device](vdev);
}
Thursday, January 5, 12 40

41. e
icfig
Dut c evon Virtio Device
Newabo
static void virtio_[device]_ get_config (VirtIODevice *vdev,
uint8_t *config)
{
VirtIO[device] *n = to_virtio_[device](vdev);
struct virtio_[device]_config cfg;
stw_p(&cfg.config, n->status);
memcpy(config, &cfg, sizeof(cfg));
}
static void virtio_[device]_ set_config (VirtIODevice *vdev, const
uint8_t *config)
{
VirtIO[de vice] *n = to_virtio_[device](vdev);
struct virtio_[device]_config cfg;
}
Thursday, January 5, 12 41

42. icee
eves t
D& r
w set
Virtio Device
Ne
static void virtio_[device]_ reset (VirtIODevice *vdev)
{
}
static void virtio_[device]_ set_status (VirtIODevice *vdev,
uint8_t val)
{
VirtIO[device] *n = to_virtio_my(vdev);
}
Thursday, January 5, 12 42

43. e
icice
Dial de ev v Virtio Device
New init
VirtIODevice *virtio_[device]_init(DeviceState *dev) {
VirtIO[device] *s;
s = (VirtIO[device]*)virtio_common_init("virtio-[device]",
VIRTIO_ID_[device], sizeof(struct virtio_[device]_config),
sizeof(VirtIO[device]));
s->vdev.get_config = virtio_[device]get_config;
s->vdev.set_config = virtio_[device]_set_config;
s->vdev.get_features = virtio_[device]_get_features;
s->vdev.set_features = virtio_[device]_set_features;
s->vdev.bad_features = virtio_[device]_bad_features;
s->vdev.reset = virtio_[device]_reset;
s->vdev.set_status = virtio_[device]_set_status;
.....
Thursday, January 5, 12 43

44. e
icice
Dial de ev v Virtio Device
New init
.....
s->vdev.set_status = virtio_[device]_set_status;
s->vq[0] = virtio_add_queue(&s->vdev, 1024,
virtio_[device]_handle_out1);
s->vq[1] = virtio_add_queue(&s->vdev, 1024,
virtio_[device]_handle_out2);
s->qdev = dev;
s->status = 99;
register_savevm(dev, "virtio-[device]”, 0, 2,
virtio_[device]_save, virtio_[device]_load, s);
return &s->vdev;
}
Thursday, January 5, 12 44

45. ivcce
ev i e
Dit de Virtio Device
New ex
void virtio_[device]_exit (VirtIODevice *vdev)
{
VirtIO[device] *s = to_virtio_[device](vdev);
unregister_savevm(s->qdev, "virtio-[device]“ , s);
}
Thursday, January 5, 12 45

46. e iloae
v cd
De & Virtio Device
New sav
static void virtio_[device]_save(QEMUFile *f, void *opaque)
{
VirtIO[device] *n = opaque;
virtio_save(&n->vdev, f);
}
static int virtio_[device]_load(QEMUFile *f, void *opaque, int
version_id)
{
VirtIO[device] *n = opaque;
virtio_load(&n->vdev, f);
return 0;
}
Thursday, January 5, 12 46

47. icler
ev d e
Da han Virtio Device
Newdat
static void virtio_[device]_handle_out(VirtIODevice *vdev,
VirtQueue *vq)
{
VirtIO[device] *n = to_virtio_my(vdev);
VirtQueueElement elem1, elem2;
int K;
memset(&elem1, 0, sizeof(elem1));
memset(&elem2, 0, sizeof(elem2));
virtqueue_pop(vq, &elem1);
if(memcmp(&elem1, &elem2, sizeof(elem1)))
printf("something happened!!!n");
K = *((int*)elem1.out_sg[0].iov_base) + 1;
memcpy(elem1.in_sg[0].iov_base, &K, sizeof(int));
virtqueue_push(vq, &elem1, sizeof(int));
}
Thursday, January 5, 12 47

48. tu re
hit ec
A rc
Virtio Driver
Guest
Virtio PCI Controller
Vring Transport
Virtio PCI Controller
QEMU
Virtio Device
Thursday, January 5, 12 48

49. r
eid
Devic rive Virtio Driver
New d
Modify kernel-src/include/linux/virtio-ids.h
#define VIRTIO_ID_[device] number
Should be the same number as
defined in qemu
Thursday, January 5, 12 49

50. river
D Virtio Driver
New
#include "linux/virtio.h“
#include "linux/virtio_ids.h“
#include "linux/virtio_pci.h“
#include "virtio.h"
static struct virtio_device_id id_table[] = {
{VIRTIO_ID_[device], VIRTIO_DEV_ANY_ID}, {0}
};
struct virtio_[device]{
int size;
void* data;
void* mapbuf;
struct semaphore sem;
struct cdev cdev;
struct virtio_device *vdev;
struct virtqueue *vq[2];
} my_cdev;
Thursday, January 5, 12 50

51. ecrure
riv u t
De str Virtio Driver
ewefin
N d
static struct virtio_driver virtio_[device]_driver = {
.feature_table = features,
.feature_table_size = ARRAY_SIZE(features),
.driver.name = KBUILD_MODNAME,
.driver.owner = THIS_MODULE,
.id_table = id_table,
.probe = my_probe,
.remove = __devexit_p(my_remove),
};
Thursday, January 5, 12 51

52. river
D Virtio Driver
New
Register & unregister driver - two APIs
register_virtio_driver(&virtio_[device]_driver);
unregister_virtio_driver(&virtio_[device]_driver);
Thursday, January 5, 12 52

53. tu re
hit ec
A rc
Virtio Driver
Guest
Virtio PCI Controller
Vring Transport
Virtio PCI Controller
QEMU
Virtio Device
Thursday, January 5, 12 53

54. ati on
nviec side
uri r Vring
omm d
C
Declare necessary parameter
char * buffer1, buffer2;
int* A[2] = {buffer1, buffer2};
struct scatterlist sg[2]; /*0 for output, 1 for input*/
struct virtqueue *vq[2];
const char *names[2] = { “[Q1]“ , “[Q2]“ };
vq_callback_t *cb[2] = {NULL, NULL};
Find virtqueue
vdev->config->find_vqs(vdev, 2, vq, cb, names);
my_cdev.vq[0] = vq[0];
my_cdev.vq[1] = vq[1];
Thursday, January 5, 12 54

55. ati on
nviec side
uri r Vring
omm d
C
Send data to qemu
sg_init_table(sg, ARRAY_SIZE(sg));
sg_set_buf(&sg[0], buffer1, [count]);
sg_set_buf(&sg[1], buffer2, [count]);
virtqueue_add_buf(myc->vq[0], sg, /*out*/1, /*in*/1, A);
virtqueue_kick(myc->vq[0]);
Get data from qemu
virtqueue_get_buf(myc->vq[0], &len);
Thursday, January 5, 12 55

56. ati on
niicc side
uev e Vring
omm d
C
Set virtqueue handler @init function
s->vq[0] = virtio_add_queue(&s->vdev, 1024,
virtio_[device]_handle_out1);
s->vq[1] = virtio_add_queue(&s->vdev, 1024,
virtio_[device]_handle_out2);
Declare variable for data @virtqueue handler
VirtQueueElement elem1, elem2;
memset(&elem1, 0, sizeof(elem1));
memset(&elem2, 0, sizeof(elem2));
Thursday, January 5, 12 56

57. ati on
niicc side
uev e Vring
omm d
C
VirtQueueElement
typedef struct VirtQueueElement
{
unsigned int index;
unsigned int out_num;
unsigned int in_num;
target_phys_addr_t in_addr[VIRTQUEUE_MAX_SIZE];
target_phys_addr_t out_addr[VIRTQUEUE_MAX_SIZE];
struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
} VirtQueueElement;
Thursday, January 5, 12 57

58. ati on
niicc side
uev e Vring
omm d
C
Pop
virtqueue_pop(vq, &elem1);
Check if data exist
if(memcmp(&elem1, &elem2, sizeof(elem1)))
do something;
Put data to elem
memcpy(elem1.in_sg[0].iov_base, &data, sizeof(data));
Push data back to guest
elem1.in_sg[0].iov_len = sizeof(data);
virtqueue_push(vq, &elem1, sizeof(data));
Thursday, January 5, 12 58

59. en ce
R e fer
Virtio: An I/O virtualization framework for Linux
http://www.ibm.com/developerworks/linux/library/l-virtio/
index.html?ca=dgr-lnxw97Viriodth-
LX&S_TACT=105AGX59&S_CMP=grlnxw97
virtio: Towards a De-Facto Standard For Virtual IO Devices
http://www.tdeig.ch/kvm/pasche/32_virtio_Russel.pdf
QEMU & Kernel Source Code
http://kernel.org/
http://wiki.qemu.org/
Thursday, January 5, 12 59