kvm-all.c 52.3 KB
Newer Older
aliguori's avatar
aliguori committed
1 2 3 4
/*
 * QEMU KVM support
 *
 * Copyright IBM, Corp. 2008
5
 *           Red Hat, Inc. 2008
aliguori's avatar
aliguori committed
6 7 8
 *
 * Authors:
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *  Glauber Costa     <gcosta@redhat.com>
aliguori's avatar
aliguori committed
10 11 12 13 14 15 16 17 18
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 *
 */

#include <sys/types.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
19
#include <stdarg.h>
aliguori's avatar
aliguori committed
20 21 22 23

#include <linux/kvm.h>

#include "qemu-common.h"
24 25 26
#include "qemu/atomic.h"
#include "qemu/option.h"
#include "qemu/config-file.h"
27
#include "sysemu/sysemu.h"
Jan Kiszka's avatar
Jan Kiszka committed
28
#include "hw/hw.h"
29
#include "hw/pci/msi.h"
30
#include "exec/gdbstub.h"
31
#include "sysemu/kvm.h"
32
#include "qemu/bswap.h"
33 34
#include "exec/memory.h"
#include "exec/address-spaces.h"
35
#include "qemu/event_notifier.h"
36
#include "trace.h"
aliguori's avatar
aliguori committed
37

38 39 40 41 42
/* This check must be after config-host.h is included */
#ifdef CONFIG_EVENTFD
#include <sys/eventfd.h>
#endif

43 44 45 46
#ifdef CONFIG_VALGRIND_H
#include <valgrind/memcheck.h>
#endif

47
/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
aliguori's avatar
aliguori committed
48 49
#define PAGE_SIZE TARGET_PAGE_SIZE

aliguori's avatar
aliguori committed
50 51 52
//#define DEBUG_KVM

#ifdef DEBUG_KVM
53
#define DPRINTF(fmt, ...) \
aliguori's avatar
aliguori committed
54 55
    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
#else
56
#define DPRINTF(fmt, ...) \
aliguori's avatar
aliguori committed
57 58 59
    do { } while (0)
#endif

60 61
#define KVM_MSI_HASHTAB_SIZE    256

aliguori's avatar
aliguori committed
62 63
typedef struct KVMSlot
{
64
    hwaddr start_addr;
65
    ram_addr_t memory_size;
66
    void *ram;
aliguori's avatar
aliguori committed
67 68 69
    int slot;
    int flags;
} KVMSlot;
aliguori's avatar
aliguori committed
70

71 72
typedef struct kvm_dirty_log KVMDirtyLog;

aliguori's avatar
aliguori committed
73 74 75 76 77
struct KVMState
{
    KVMSlot slots[32];
    int fd;
    int vmfd;
aliguori's avatar
aliguori committed
78
    int coalesced_mmio;
79
    struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
80
    bool coalesced_flush_in_progress;
81
    int broken_set_mem_region;
82
    int migration_log;
83
    int vcpu_events;
84
    int robust_singlestep;
85
    int debugregs;
86 87 88
#ifdef KVM_CAP_SET_GUEST_DEBUG
    struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
#endif
89
    int pit_state2;
90
    int xsave, xcrs;
91
    int many_ioeventfds;
92
    int intx_set_mask;
93 94 95
    /* The man page (and posix) say ioctl numbers are signed int, but
     * they're not.  Linux, glibc and *BSD all treat ioctl numbers as
     * unsigned, and treating them as signed here can break things */
96
    unsigned irq_set_ioctl;
97 98 99 100
#ifdef KVM_CAP_IRQ_ROUTING
    struct kvm_irq_routing *irq_routes;
    int nr_allocated_irq_routes;
    uint32_t *used_gsi_bitmap;
101
    unsigned int gsi_count;
102
    QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
103
    bool direct_msi;
104
#endif
aliguori's avatar
aliguori committed
105 106
};

107
KVMState *kvm_state;
108
bool kvm_kernel_irqchip;
109
bool kvm_async_interrupts_allowed;
110
bool kvm_halt_in_kernel_allowed;
111
bool kvm_irqfds_allowed;
112
bool kvm_msi_via_irqfd_allowed;
113
bool kvm_gsi_routing_allowed;
114
bool kvm_allowed;
115
bool kvm_readonly_mem_allowed;
aliguori's avatar
aliguori committed
116

117 118 119 120 121 122
static const KVMCapabilityInfo kvm_required_capabilites[] = {
    KVM_CAP_INFO(USER_MEMORY),
    KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
    KVM_CAP_LAST_INFO
};

aliguori's avatar
aliguori committed
123 124 125 126 127
static KVMSlot *kvm_alloc_slot(KVMState *s)
{
    int i;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
128
        if (s->slots[i].memory_size == 0) {
aliguori's avatar
aliguori committed
129
            return &s->slots[i];
130
        }
aliguori's avatar
aliguori committed
131 132
    }

133 134 135 136 137
    fprintf(stderr, "%s: no free slot available\n", __func__);
    abort();
}

static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
138 139
                                         hwaddr start_addr,
                                         hwaddr end_addr)
140 141 142 143 144 145 146 147 148 149 150 151
{
    int i;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
        KVMSlot *mem = &s->slots[i];

        if (start_addr == mem->start_addr &&
            end_addr == mem->start_addr + mem->memory_size) {
            return mem;
        }
    }

aliguori's avatar
aliguori committed
152 153 154
    return NULL;
}

155 156 157 158
/*
 * Find overlapping slot with lowest start address
 */
static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
159 160
                                            hwaddr start_addr,
                                            hwaddr end_addr)
aliguori's avatar
aliguori committed
161
{
162
    KVMSlot *found = NULL;
aliguori's avatar
aliguori committed
163 164 165 166 167
    int i;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
        KVMSlot *mem = &s->slots[i];

168 169 170 171 172 173 174 175 176
        if (mem->memory_size == 0 ||
            (found && found->start_addr < mem->start_addr)) {
            continue;
        }

        if (end_addr > mem->start_addr &&
            start_addr < mem->start_addr + mem->memory_size) {
            found = mem;
        }
aliguori's avatar
aliguori committed
177 178
    }

179
    return found;
aliguori's avatar
aliguori committed
180 181
}

182
int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
183
                                       hwaddr *phys_addr)
184 185 186 187 188 189
{
    int i;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
        KVMSlot *mem = &s->slots[i];

190 191
        if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
            *phys_addr = mem->start_addr + (ram - mem->ram);
192 193 194 195 196 197 198
            return 1;
        }
    }

    return 0;
}

199 200 201 202 203 204
static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
{
    struct kvm_userspace_memory_region mem;

    mem.slot = slot->slot;
    mem.guest_phys_addr = slot->start_addr;
205
    mem.userspace_addr = (unsigned long)slot->ram;
206
    mem.flags = slot->flags;
207 208 209
    if (s->migration_log) {
        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
    }
210 211

    if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
212 213 214 215 216 217
        /* Set the slot size to 0 before setting the slot to the desired
         * value. This is needed based on KVM commit 75d61fbc. */
        mem.memory_size = 0;
        kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
    }
    mem.memory_size = slot->memory_size;
218 219 220
    return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
}

221 222
static void kvm_reset_vcpu(void *opaque)
{
223
    CPUState *cpu = opaque;
224

225
    kvm_arch_reset_vcpu(cpu);
226
}
227

228
int kvm_init_vcpu(CPUState *cpu)
aliguori's avatar
aliguori committed
229 230 231 232 233
{
    KVMState *s = kvm_state;
    long mmap_size;
    int ret;

234
    DPRINTF("kvm_init_vcpu\n");
aliguori's avatar
aliguori committed
235

236
    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
aliguori's avatar
aliguori committed
237
    if (ret < 0) {
238
        DPRINTF("kvm_create_vcpu failed\n");
aliguori's avatar
aliguori committed
239 240 241
        goto err;
    }

242
    cpu->kvm_fd = ret;
243
    cpu->kvm_state = s;
244
    cpu->kvm_vcpu_dirty = true;
aliguori's avatar
aliguori committed
245 246 247

    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
    if (mmap_size < 0) {
248
        ret = mmap_size;
249
        DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
aliguori's avatar
aliguori committed
250 251 252
        goto err;
    }

253
    cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
254
                        cpu->kvm_fd, 0);
255
    if (cpu->kvm_run == MAP_FAILED) {
aliguori's avatar
aliguori committed
256
        ret = -errno;
257
        DPRINTF("mmap'ing vcpu state failed\n");
aliguori's avatar
aliguori committed
258 259 260
        goto err;
    }

261 262
    if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
        s->coalesced_mmio_ring =
263
            (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
264
    }
265

266
    ret = kvm_arch_init_vcpu(cpu);
267
    if (ret == 0) {
268 269
        qemu_register_reset(kvm_reset_vcpu, cpu);
        kvm_arch_reset_vcpu(cpu);
270
    }
aliguori's avatar
aliguori committed
271 272 273 274
err:
    return ret;
}

275 276 277
/*
 * dirty pages logging control
 */
278

279
static int kvm_mem_flags(KVMState *s, bool log_dirty, bool readonly)
280
{
281 282 283 284 285 286
    int flags = 0;
    flags = log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
    if (readonly && kvm_readonly_mem_allowed) {
        flags |= KVM_MEM_READONLY;
    }
    return flags;
287 288 289
}

static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
290 291
{
    KVMState *s = kvm_state;
292
    int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
293 294 295
    int old_flags;

    old_flags = mem->flags;
296

297
    flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty, false);
298 299
    mem->flags = flags;

300 301 302 303
    /* If nothing changed effectively, no need to issue ioctl */
    if (s->migration_log) {
        flags |= KVM_MEM_LOG_DIRTY_PAGES;
    }
304

305
    if (flags == old_flags) {
306
        return 0;
307 308
    }

309 310 311
    return kvm_set_user_memory_region(s, mem);
}

312
static int kvm_dirty_pages_log_change(hwaddr phys_addr,
313 314 315 316 317 318 319 320
                                      ram_addr_t size, bool log_dirty)
{
    KVMState *s = kvm_state;
    KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);

    if (mem == NULL)  {
        fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
                TARGET_FMT_plx "\n", __func__, phys_addr,
321
                (hwaddr)(phys_addr + size - 1));
322 323 324 325 326
        return -EINVAL;
    }
    return kvm_slot_dirty_pages_log_change(mem, log_dirty);
}

327 328
static void kvm_log_start(MemoryListener *listener,
                          MemoryRegionSection *section)
329
{
330 331 332
    int r;

    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
333
                                   int128_get64(section->size), true);
334 335 336
    if (r < 0) {
        abort();
    }
337 338
}

339 340
static void kvm_log_stop(MemoryListener *listener,
                          MemoryRegionSection *section)
341
{
342 343 344
    int r;

    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
345
                                   int128_get64(section->size), false);
346 347 348
    if (r < 0) {
        abort();
    }
349 350
}

351
static int kvm_set_migration_log(int enable)
352 353 354 355 356 357 358 359 360 361
{
    KVMState *s = kvm_state;
    KVMSlot *mem;
    int i, err;

    s->migration_log = enable;

    for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
        mem = &s->slots[i];

362 363 364
        if (!mem->memory_size) {
            continue;
        }
365 366 367 368 369 370 371 372 373 374 375
        if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
            continue;
        }
        err = kvm_set_user_memory_region(s, mem);
        if (err) {
            return err;
        }
    }
    return 0;
}

376
/* get kvm's dirty pages bitmap and update qemu's */
377 378
static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
                                         unsigned long *bitmap)
379
{
380
    unsigned int i, j;
381
    unsigned long page_number, c;
382
    hwaddr addr, addr1;
383 384
    unsigned int pages = int128_get64(section->size) / getpagesize();
    unsigned int len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
385
    unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
386 387 388 389 390 391 392 393 394 395 396

    /*
     * bitmap-traveling is faster than memory-traveling (for addr...)
     * especially when most of the memory is not dirty.
     */
    for (i = 0; i < len; i++) {
        if (bitmap[i] != 0) {
            c = leul_to_cpu(bitmap[i]);
            do {
                j = ffsl(c) - 1;
                c &= ~(1ul << j);
397
                page_number = (i * HOST_LONG_BITS + j) * hpratio;
398
                addr1 = page_number * TARGET_PAGE_SIZE;
399
                addr = section->offset_within_region + addr1;
400 401
                memory_region_set_dirty(section->mr, addr,
                                        TARGET_PAGE_SIZE * hpratio);
402 403 404 405
            } while (c != 0);
        }
    }
    return 0;
406 407
}

408 409
#define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))

410 411
/**
 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
412 413 414
 * This function updates qemu's dirty bitmap using
 * memory_region_set_dirty().  This means all bits are set
 * to dirty.
415
 *
416
 * @start_add: start of logged region.
417 418
 * @end_addr: end of logged region.
 */
419
static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
420 421
{
    KVMState *s = kvm_state;
422 423 424 425
    unsigned long size, allocated_size = 0;
    KVMDirtyLog d;
    KVMSlot *mem;
    int ret = 0;
426
    hwaddr start_addr = section->offset_within_address_space;
427
    hwaddr end_addr = start_addr + int128_get64(section->size);
428

429 430 431 432 433 434
    d.dirty_bitmap = NULL;
    while (start_addr < end_addr) {
        mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
        if (mem == NULL) {
            break;
        }
435

436 437 438 439 440 441 442 443 444 445 446 447 448 449
        /* XXX bad kernel interface alert
         * For dirty bitmap, kernel allocates array of size aligned to
         * bits-per-long.  But for case when the kernel is 64bits and
         * the userspace is 32bits, userspace can't align to the same
         * bits-per-long, since sizeof(long) is different between kernel
         * and user space.  This way, userspace will provide buffer which
         * may be 4 bytes less than the kernel will use, resulting in
         * userspace memory corruption (which is not detectable by valgrind
         * too, in most cases).
         * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
         * a hope that sizeof(long) wont become >8 any time soon.
         */
        size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
                     /*HOST_LONG_BITS*/ 64) / 8;
450
        if (!d.dirty_bitmap) {
451
            d.dirty_bitmap = g_malloc(size);
452
        } else if (size > allocated_size) {
453
            d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
454 455 456
        }
        allocated_size = size;
        memset(d.dirty_bitmap, 0, allocated_size);
457

458
        d.slot = mem->slot;
459

460
        if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
461
            DPRINTF("ioctl failed %d\n", errno);
462 463 464
            ret = -1;
            break;
        }
465

466
        kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
467
        start_addr = mem->start_addr + mem->memory_size;
468
    }
469
    g_free(d.dirty_bitmap);
470 471

    return ret;
472 473
}

474 475
static void kvm_coalesce_mmio_region(MemoryListener *listener,
                                     MemoryRegionSection *secion,
476
                                     hwaddr start, hwaddr size)
aliguori's avatar
aliguori committed
477 478 479 480 481 482 483 484
{
    KVMState *s = kvm_state;

    if (s->coalesced_mmio) {
        struct kvm_coalesced_mmio_zone zone;

        zone.addr = start;
        zone.size = size;
485
        zone.pad = 0;
aliguori's avatar
aliguori committed
486

487
        (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
aliguori's avatar
aliguori committed
488 489 490
    }
}

491 492
static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
                                       MemoryRegionSection *secion,
493
                                       hwaddr start, hwaddr size)
aliguori's avatar
aliguori committed
494 495 496 497 498 499 500 501
{
    KVMState *s = kvm_state;

    if (s->coalesced_mmio) {
        struct kvm_coalesced_mmio_zone zone;

        zone.addr = start;
        zone.size = size;
502
        zone.pad = 0;
aliguori's avatar
aliguori committed
503

504
        (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
aliguori's avatar
aliguori committed
505 506 507
    }
}

508 509 510 511 512 513 514 515 516 517 518 519
int kvm_check_extension(KVMState *s, unsigned int extension)
{
    int ret;

    ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
    if (ret < 0) {
        ret = 0;
    }

    return ret;
}

520
static int kvm_set_ioeventfd_mmio(int fd, uint32_t addr, uint32_t val,
521
                                  bool assign, uint32_t size, bool datamatch)
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
522 523 524 525
{
    int ret;
    struct kvm_ioeventfd iofd;

526
    iofd.datamatch = datamatch ? val : 0;
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
527 528
    iofd.addr = addr;
    iofd.len = size;
529
    iofd.flags = 0;
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
530 531 532 533 534 535
    iofd.fd = fd;

    if (!kvm_enabled()) {
        return -ENOSYS;
    }

536 537 538
    if (datamatch) {
        iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
    }
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
539 540 541 542 543 544 545 546 547 548 549 550 551
    if (!assign) {
        iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
    }

    ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);

    if (ret < 0) {
        return -errno;
    }

    return 0;
}

552
static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
553
                                 bool assign, uint32_t size, bool datamatch)
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
554 555
{
    struct kvm_ioeventfd kick = {
556
        .datamatch = datamatch ? val : 0,
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
557
        .addr = addr,
558
        .flags = KVM_IOEVENTFD_FLAG_PIO,
559
        .len = size,
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
560 561 562 563 564 565
        .fd = fd,
    };
    int r;
    if (!kvm_enabled()) {
        return -ENOSYS;
    }
566 567 568
    if (datamatch) {
        kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
    }
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
569 570 571 572 573 574 575 576 577 578 579
    if (!assign) {
        kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
    }
    r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
    if (r < 0) {
        return r;
    }
    return 0;
}


580 581
static int kvm_check_many_ioeventfds(void)
{
582 583 584 585 586
    /* Userspace can use ioeventfd for io notification.  This requires a host
     * that supports eventfd(2) and an I/O thread; since eventfd does not
     * support SIGIO it cannot interrupt the vcpu.
     *
     * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
587 588
     * can avoid creating too many ioeventfds.
     */
589
#if defined(CONFIG_EVENTFD)
590 591 592 593 594 595 596
    int ioeventfds[7];
    int i, ret = 0;
    for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
        ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
        if (ioeventfds[i] < 0) {
            break;
        }
597
        ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
598 599 600 601 602 603 604 605 606 607
        if (ret < 0) {
            close(ioeventfds[i]);
            break;
        }
    }

    /* Decide whether many devices are supported or not */
    ret = i == ARRAY_SIZE(ioeventfds);

    while (i-- > 0) {
608
        kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
609 610 611 612 613 614 615 616
        close(ioeventfds[i]);
    }
    return ret;
#else
    return 0;
#endif
}

617 618 619 620 621 622 623 624 625 626 627 628
static const KVMCapabilityInfo *
kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
{
    while (list->name) {
        if (!kvm_check_extension(s, list->value)) {
            return list;
        }
        list++;
    }
    return NULL;
}

629
static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
630 631 632 633
{
    KVMState *s = kvm_state;
    KVMSlot *mem, old;
    int err;
634 635
    MemoryRegion *mr = section->mr;
    bool log_dirty = memory_region_is_logging(mr);
636 637
    bool writeable = !mr->readonly && !mr->rom_device;
    bool readonly_flag = mr->readonly || memory_region_is_romd(mr);
638
    hwaddr start_addr = section->offset_within_address_space;
639
    ram_addr_t size = int128_get64(section->size);
640
    void *ram = NULL;
Avi Kivity's avatar
Avi Kivity committed
641
    unsigned delta;
642

643 644
    /* kvm works in page size chunks, but the function may be called
       with sub-page size and unaligned start address. */
Avi Kivity's avatar
Avi Kivity committed
645 646 647 648 649 650 651 652 653 654
    delta = TARGET_PAGE_ALIGN(size) - size;
    if (delta > size) {
        return;
    }
    start_addr += delta;
    size -= delta;
    size &= TARGET_PAGE_MASK;
    if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
        return;
    }
655

656
    if (!memory_region_is_ram(mr)) {
657 658 659 660 661 662 663
        if (writeable || !kvm_readonly_mem_allowed) {
            return;
        } else if (!mr->romd_mode) {
            /* If the memory device is not in romd_mode, then we actually want
             * to remove the kvm memory slot so all accesses will trap. */
            add = false;
        }
664 665
    }

Avi Kivity's avatar
Avi Kivity committed
666
    ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
667

668 669 670 671 672 673
    while (1) {
        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
        if (!mem) {
            break;
        }

674
        if (add && start_addr >= mem->start_addr &&
675
            (start_addr + size <= mem->start_addr + mem->memory_size) &&
676
            (ram - start_addr == mem->ram - mem->start_addr)) {
677
            /* The new slot fits into the existing one and comes with
678 679
             * identical parameters - update flags and done. */
            kvm_slot_dirty_pages_log_change(mem, log_dirty);
680 681 682 683 684
            return;
        }

        old = *mem;

685 686 687 688
        if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
            kvm_physical_sync_dirty_bitmap(section);
        }

689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
        /* unregister the overlapping slot */
        mem->memory_size = 0;
        err = kvm_set_user_memory_region(s, mem);
        if (err) {
            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
                    __func__, strerror(-err));
            abort();
        }

        /* Workaround for older KVM versions: we can't join slots, even not by
         * unregistering the previous ones and then registering the larger
         * slot. We have to maintain the existing fragmentation. Sigh.
         *
         * This workaround assumes that the new slot starts at the same
         * address as the first existing one. If not or if some overlapping
         * slot comes around later, we will fail (not seen in practice so far)
         * - and actually require a recent KVM version. */
        if (s->broken_set_mem_region &&
707
            old.start_addr == start_addr && old.memory_size < size && add) {
708 709 710
            mem = kvm_alloc_slot(s);
            mem->memory_size = old.memory_size;
            mem->start_addr = old.start_addr;
711
            mem->ram = old.ram;
712
            mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
713 714 715 716 717 718 719 720 721

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
                        strerror(-err));
                abort();
            }

            start_addr += old.memory_size;
722
            ram += old.memory_size;
723 724 725 726 727 728 729 730 731
            size -= old.memory_size;
            continue;
        }

        /* register prefix slot */
        if (old.start_addr < start_addr) {
            mem = kvm_alloc_slot(s);
            mem->memory_size = start_addr - old.start_addr;
            mem->start_addr = old.start_addr;
732
            mem->ram = old.ram;
733
            mem->flags =  kvm_mem_flags(s, log_dirty, readonly_flag);
734 735 736 737 738

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error registering prefix slot: %s\n",
                        __func__, strerror(-err));
739 740 741 742 743
#ifdef TARGET_PPC
                fprintf(stderr, "%s: This is probably because your kernel's " \
                                "PAGE_SIZE is too big. Please try to use 4k " \
                                "PAGE_SIZE!\n", __func__);
#endif
744 745 746 747 748 749 750 751 752 753 754 755
                abort();
            }
        }

        /* register suffix slot */
        if (old.start_addr + old.memory_size > start_addr + size) {
            ram_addr_t size_delta;

            mem = kvm_alloc_slot(s);
            mem->start_addr = start_addr + size;
            size_delta = mem->start_addr - old.start_addr;
            mem->memory_size = old.memory_size - size_delta;
756
            mem->ram = old.ram + size_delta;
757
            mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
758 759 760 761 762 763 764 765 766 767 768

            err = kvm_set_user_memory_region(s, mem);
            if (err) {
                fprintf(stderr, "%s: error registering suffix slot: %s\n",
                        __func__, strerror(-err));
                abort();
            }
        }
    }

    /* in case the KVM bug workaround already "consumed" the new slot */
769
    if (!size) {
770
        return;
771
    }
772
    if (!add) {
773
        return;
774
    }
775 776 777
    mem = kvm_alloc_slot(s);
    mem->memory_size = size;
    mem->start_addr = start_addr;
778
    mem->ram = ram;
779
    mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
780 781 782 783 784 785 786 787 788

    err = kvm_set_user_memory_region(s, mem);
    if (err) {
        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
                strerror(-err));
        abort();
    }
}

789 790 791
static void kvm_region_add(MemoryListener *listener,
                           MemoryRegionSection *section)
{
Paolo Bonzini's avatar
Paolo Bonzini committed
792
    memory_region_ref(section->mr);
793 794 795 796 797 798 799
    kvm_set_phys_mem(section, true);
}

static void kvm_region_del(MemoryListener *listener,
                           MemoryRegionSection *section)
{
    kvm_set_phys_mem(section, false);
Paolo Bonzini's avatar
Paolo Bonzini committed
800
    memory_region_unref(section->mr);
801 802 803 804
}

static void kvm_log_sync(MemoryListener *listener,
                         MemoryRegionSection *section)
805
{
806 807
    int r;

808
    r = kvm_physical_sync_dirty_bitmap(section);
809 810 811
    if (r < 0) {
        abort();
    }
812 813
}

814
static void kvm_log_global_start(struct MemoryListener *listener)
815
{
816 817 818 819
    int r;

    r = kvm_set_migration_log(1);
    assert(r >= 0);
820 821
}

822
static void kvm_log_global_stop(struct MemoryListener *listener)
823
{
824 825 826 827
    int r;

    r = kvm_set_migration_log(0);
    assert(r >= 0);
828 829
}

830 831 832 833 834 835
static void kvm_mem_ioeventfd_add(MemoryListener *listener,
                                  MemoryRegionSection *section,
                                  bool match_data, uint64_t data,
                                  EventNotifier *e)
{
    int fd = event_notifier_get_fd(e);
836 837
    int r;

838
    r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
839 840
                               data, true, int128_get64(section->size),
                               match_data);
841
    if (r < 0) {
842 843
        fprintf(stderr, "%s: error adding ioeventfd: %s\n",
                __func__, strerror(-r));
844 845 846 847
        abort();
    }
}

848 849 850 851
static void kvm_mem_ioeventfd_del(MemoryListener *listener,
                                  MemoryRegionSection *section,
                                  bool match_data, uint64_t data,
                                  EventNotifier *e)
852
{
853
    int fd = event_notifier_get_fd(e);
854 855
    int r;

856
    r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
857 858
                               data, false, int128_get64(section->size),
                               match_data);
859 860 861 862 863
    if (r < 0) {
        abort();
    }
}

864 865 866 867
static void kvm_io_ioeventfd_add(MemoryListener *listener,
                                 MemoryRegionSection *section,
                                 bool match_data, uint64_t data,
                                 EventNotifier *e)
868
{
869
    int fd = event_notifier_get_fd(e);
870 871
    int r;

872
    r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
873 874
                              data, true, int128_get64(section->size),
                              match_data);
875
    if (r < 0) {
876 877
        fprintf(stderr, "%s: error adding ioeventfd: %s\n",
                __func__, strerror(-r));
878 879 880 881
        abort();
    }
}

882 883 884 885
static void kvm_io_ioeventfd_del(MemoryListener *listener,
                                 MemoryRegionSection *section,
                                 bool match_data, uint64_t data,
                                 EventNotifier *e)
886 887

{
888
    int fd = event_notifier_get_fd(e);
889 890
    int r;

891
    r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
892 893
                              data, false, int128_get64(section->size),
                              match_data);
894 895 896 897 898
    if (r < 0) {
        abort();
    }
}

899 900 901
static MemoryListener kvm_memory_listener = {
    .region_add = kvm_region_add,
    .region_del = kvm_region_del,
902 903
    .log_start = kvm_log_start,
    .log_stop = kvm_log_stop,
904 905 906
    .log_sync = kvm_log_sync,
    .log_global_start = kvm_log_global_start,
    .log_global_stop = kvm_log_global_stop,
907 908
    .eventfd_add = kvm_mem_ioeventfd_add,
    .eventfd_del = kvm_mem_ioeventfd_del,
909 910
    .coalesced_mmio_add = kvm_coalesce_mmio_region,
    .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
911 912 913 914 915 916
    .priority = 10,
};

static MemoryListener kvm_io_listener = {
    .eventfd_add = kvm_io_ioeventfd_add,
    .eventfd_del = kvm_io_ioeventfd_del,
917
    .priority = 10,
918 919
};

920
static void kvm_handle_interrupt(CPUState *cpu, int mask)
921
{
922
    cpu->interrupt_request |= mask;
923

924
    if (!qemu_cpu_is_self(cpu)) {
925
        qemu_cpu_kick(cpu);
926 927 928
    }
}

929
int kvm_set_irq(KVMState *s, int irq, int level)
930 931 932 933
{
    struct kvm_irq_level event;
    int ret;

934
    assert(kvm_async_interrupts_enabled());
935 936 937

    event.level = level;
    event.irq = irq;
938
    ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
939
    if (ret < 0) {
940
        perror("kvm_set_irq");
941 942 943
        abort();
    }

944
    return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
945 946 947
}

#ifdef KVM_CAP_IRQ_ROUTING
948 949 950 951 952
typedef struct KVMMSIRoute {
    struct kvm_irq_routing_entry kroute;
    QTAILQ_ENTRY(KVMMSIRoute) entry;
} KVMMSIRoute;

953 954 955 956 957
static void set_gsi(KVMState *s, unsigned int gsi)
{
    s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
}

958 959 960 961 962
static void clear_gsi(KVMState *s, unsigned int gsi)
{
    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
}

963
void kvm_init_irq_routing(KVMState *s)
964
{
965
    int gsi_count, i;
966 967 968 969 970 971

    gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
    if (gsi_count > 0) {
        unsigned int gsi_bits, i;

        /* Round up so we can search ints using ffs */
972
        gsi_bits = ALIGN(gsi_count, 32);
973
        s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
974
        s->gsi_count = gsi_count;
975 976 977 978 979 980 981 982 983 984

        /* Mark any over-allocated bits as already in use */
        for (i = gsi_count; i < gsi_bits; i++) {
            set_gsi(s, i);
        }
    }

    s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
    s->nr_allocated_irq_routes = 0;

985 986 987 988
    if (!s->direct_msi) {
        for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
            QTAILQ_INIT(&s->msi_hashtab[i]);
        }
989 990
    }

991 992 993
    kvm_arch_init_irq_routing(s);
}

994
void kvm_irqchip_commit_routes(KVMState *s)
995 996 997 998 999 1000 1001 1002
{
    int ret;

    s->irq_routes->flags = 0;
    ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
    assert(ret == 0);
}

1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
static void kvm_add_routing_entry(KVMState *s,
                                  struct kvm_irq_routing_entry *entry)
{
    struct kvm_irq_routing_entry *new;
    int n, size;

    if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
        n = s->nr_allocated_irq_routes * 2;
        if (n < 64) {
            n = 64;
        }
        size = sizeof(struct kvm_irq_routing);
        size += n * sizeof(*new);
        s->irq_routes = g_realloc(s->irq_routes, size);
        s->nr_allocated_irq_routes = n;
    }
    n = s->irq_routes->nr++;
    new = &s->irq_routes->entries[n];
1021 1022

    *new = *entry;
1023 1024 1025 1026

    set_gsi(s, entry->gsi);
}

1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
static int kvm_update_routing_entry(KVMState *s,
                                    struct kvm_irq_routing_entry *new_entry)
{
    struct kvm_irq_routing_entry *entry;
    int n;

    for (n = 0; n < s->irq_routes->nr; n++) {
        entry = &s->irq_routes->entries[n];
        if (entry->gsi != new_entry->gsi) {
            continue;
        }

1039 1040 1041 1042
        if(!memcmp(entry, new_entry, sizeof *entry)) {
            return 0;
        }

1043
        *entry = *new_entry;
1044 1045 1046 1047 1048 1049 1050 1051 1052

        kvm_irqchip_commit_routes(s);

        return 0;
    }

    return -ESRCH;
}

1053
void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
1054
{
1055
    struct kvm_irq_routing_entry e = {};
1056

1057 1058
    assert(pin < s->gsi_count);

1059 1060 1061 1062 1063 1064 1065 1066
    e.gsi = irq;
    e.type = KVM_IRQ_ROUTING_IRQCHIP;
    e.flags = 0;
    e.u.irqchip.irqchip = irqchip;
    e.u.irqchip.pin = pin;
    kvm_add_routing_entry(s, &e);
}

1067
void kvm_irqchip_release_virq(KVMState *s, int virq)
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
{
    struct kvm_irq_routing_entry *e;
    int i;

    for (i = 0; i < s->irq_routes->nr; i++) {
        e = &s->irq_routes->entries[i];
        if (e->gsi == virq) {
            s->irq_routes->nr--;
            *e = s->irq_routes->entries[s->irq_routes->nr];
        }
    }
    clear_gsi(s, virq);
}

static unsigned int kvm_hash_msi(uint32_t data)
{
    /* This is optimized for IA32 MSI layout. However, no other arch shall
     * repeat the mistake of not providing a direct MSI injection API. */
    return data & 0xff;
}

static void kvm_flush_dynamic_msi_routes(KVMState *s)
{
    KVMMSIRoute *route, *next;
    unsigned int hash;

    for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
        QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
            kvm_irqchip_release_virq(s, route->kroute.gsi);
            QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
            g_free(route);
        }
    }
}

static int kvm_irqchip_get_virq(KVMState *s)
{
    uint32_t *word = s->used_gsi_bitmap;
    int max_words = ALIGN(s->gsi_count, 32) / 32;
    int i, bit;
    bool retry = true;

again:
    /* Return the lowest unused GSI in the bitmap */
    for (i = 0; i < max_words; i++) {
        bit = ffs(~word[i]);
        if (!bit) {
            continue;
        }

        return bit - 1 + i * 32;
    }
1120
    if (!s->direct_msi && retry) {
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
        retry = false;
        kvm_flush_dynamic_msi_routes(s);
        goto again;
    }
    return -ENOSPC;

}

static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
{
    unsigned int hash = kvm_hash_msi(msg.data);
    KVMMSIRoute *route;

    QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
        if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
            route->kroute.u.msi.address_hi == (msg.address >> 32) &&
1137
            route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
1138 1139 1140 1141 1142 1143 1144 1145
            return route;
        }
    }
    return NULL;
}

int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
{
1146
    struct kvm_msi msi;
1147 1148
    KVMMSIRoute *route;

1149 1150 1151
    if (s->direct_msi) {
        msi.address_lo = (uint32_t)msg.address;
        msi.address_hi = msg.address >> 32;
1152
        msi.data = le32_to_cpu(msg.data);
1153 1154 1155 1156 1157 1158
        msi.flags = 0;
        memset(msi.pad, 0, sizeof(msi.pad));

        return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
    }

1159 1160
    route = kvm_lookup_msi_route(s, msg);
    if (!route) {
1161
        int virq;
1162 1163 1164 1165 1166 1167

        virq = kvm_irqchip_get_virq(s);
        if (virq < 0) {
            return virq;
        }

1168
        route = g_malloc0(sizeof(KVMMSIRoute));
1169 1170 1171 1172 1173
        route->kroute.gsi = virq;
        route->kroute.type = KVM_IRQ_ROUTING_MSI;
        route->kroute.flags = 0;
        route->kroute.u.msi.address_lo = (uint32_t)msg.address;
        route->kroute.u.msi.address_hi = msg.address >> 32;
1174
        route->kroute.u.msi.data = le32_to_cpu(msg.data);
1175 1176

        kvm_add_routing_entry(s, &route->kroute);
1177
        kvm_irqchip_commit_routes(s);
1178 1179 1180 1181 1182 1183 1184

        QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
                           entry);
    }

    assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);

1185
    return kvm_set_irq(s, route->kroute.gsi, 1);
1186 1187
}

1188 1189
int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
{
1190
    struct kvm_irq_routing_entry kroute = {};
1191 1192
    int virq;