xen-hvm.c 37.2 KB
Newer Older
1 2 3 4 5 6
/*
 * Copyright (C) 2010       Citrix Ltd.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
7 8
 * Contributions after 2012-01-13 are licensed under the terms of the
 * GNU GPL, version 2 or (at your option) any later version.
9 10
 */

11 12
#include <sys/mman.h>

13
#include "hw/pci/pci.h"
14 15 16
#include "hw/i386/pc.h"
#include "hw/xen/xen_common.h"
#include "hw/xen/xen_backend.h"
17
#include "qmp-commands.h"
18

19
#include "sysemu/char.h"
20
#include "qemu/range.h"
21
#include "sysemu/xen-mapcache.h"
22
#include "trace.h"
23
#include "exec/address-spaces.h"
24

25 26
#include <xen/hvm/ioreq.h>
#include <xen/hvm/params.h>
27
#include <xen/hvm/e820.h>
28

29
//#define DEBUG_XEN_HVM
30

31
#ifdef DEBUG_XEN_HVM
32 33 34 35 36 37 38
#define DPRINTF(fmt, ...) \
    do { fprintf(stderr, "xen: " fmt, ## __VA_ARGS__); } while (0)
#else
#define DPRINTF(fmt, ...) \
    do { } while (0)
#endif

Avi Kivity's avatar
Avi Kivity committed
39
static MemoryRegion ram_memory, ram_640k, ram_lo, ram_hi;
40
static MemoryRegion *framebuffer;
41
static bool xen_in_migration;
Avi Kivity's avatar
Avi Kivity committed
42

43
/* Compatibility with older version */
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66

/* This allows QEMU to build on a system that has Xen 4.5 or earlier
 * installed.  This here (not in hw/xen/xen_common.h) because xen/hvm/ioreq.h
 * needs to be included before this block and hw/xen/xen_common.h needs to
 * be included before xen/hvm/ioreq.h
 */
#ifndef IOREQ_TYPE_VMWARE_PORT
#define IOREQ_TYPE_VMWARE_PORT  3
struct vmware_regs {
    uint32_t esi;
    uint32_t edi;
    uint32_t ebx;
    uint32_t ecx;
    uint32_t edx;
};
typedef struct vmware_regs vmware_regs_t;

struct shared_vmport_iopage {
    struct vmware_regs vcpu_vmport_regs[1];
};
typedef struct shared_vmport_iopage shared_vmport_iopage_t;
#endif

67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
#if __XEN_LATEST_INTERFACE_VERSION__ < 0x0003020a
static inline uint32_t xen_vcpu_eport(shared_iopage_t *shared_page, int i)
{
    return shared_page->vcpu_iodata[i].vp_eport;
}
static inline ioreq_t *xen_vcpu_ioreq(shared_iopage_t *shared_page, int vcpu)
{
    return &shared_page->vcpu_iodata[vcpu].vp_ioreq;
}
#  define FMT_ioreq_size PRIx64
#else
static inline uint32_t xen_vcpu_eport(shared_iopage_t *shared_page, int i)
{
    return shared_page->vcpu_ioreq[i].vp_eport;
}
static inline ioreq_t *xen_vcpu_ioreq(shared_iopage_t *shared_page, int vcpu)
{
    return &shared_page->vcpu_ioreq[vcpu];
}
#  define FMT_ioreq_size "u"
#endif
88 89 90
#ifndef HVM_PARAM_BUFIOREQ_EVTCHN
#define HVM_PARAM_BUFIOREQ_EVTCHN 26
#endif
91 92 93

#define BUFFER_IO_MAX_DELAY  100

94
typedef struct XenPhysmap {
95
    hwaddr start_addr;
96
    ram_addr_t size;
97
    const char *name;
98
    hwaddr phys_offset;
99 100 101 102

    QLIST_ENTRY(XenPhysmap) list;
} XenPhysmap;

103 104
typedef struct XenIOState {
    shared_iopage_t *shared_page;
105
    shared_vmport_iopage_t *shared_vmport_page;
106 107
    buffered_iopage_t *buffered_io_page;
    QEMUTimer *buffered_io_timer;
108
    CPUState **cpu_by_vcpu_id;
109 110
    /* the evtchn port for polling the notification, */
    evtchn_port_t *ioreq_local_port;
111 112
    /* evtchn local port for buffered io */
    evtchn_port_t bufioreq_local_port;
113 114 115 116 117
    /* the evtchn fd for polling */
    XenEvtchn xce_handle;
    /* which vcpu we are serving */
    int send_vcpu;

118
    struct xs_handle *xenstore;
119
    MemoryListener memory_listener;
120
    QLIST_HEAD(, XenPhysmap) physmap;
121
    hwaddr free_phys_offset;
122
    const XenPhysmap *log_for_dirtybit;
123

124
    Notifier exit;
125
    Notifier suspend;
126
    Notifier wakeup;
127 128
} XenIOState;

129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
/* Xen specific function for piix pci */

int xen_pci_slot_get_pirq(PCIDevice *pci_dev, int irq_num)
{
    return irq_num + ((pci_dev->devfn >> 3) << 2);
}

void xen_piix3_set_irq(void *opaque, int irq_num, int level)
{
    xc_hvm_set_pci_intx_level(xen_xc, xen_domid, 0, 0, irq_num >> 2,
                              irq_num & 3, level);
}

void xen_piix_pci_write_config_client(uint32_t address, uint32_t val, int len)
{
    int i;

    /* Scan for updates to PCI link routes (0x60-0x63). */
    for (i = 0; i < len; i++) {
        uint8_t v = (val >> (8 * i)) & 0xff;
        if (v & 0x80) {
            v = 0;
        }
        v &= 0xf;
        if (((address + i) >= 0x60) && ((address + i) <= 0x63)) {
            xc_hvm_set_pci_link_route(xen_xc, xen_domid, address + i - 0x60, v);
        }
    }
}

159 160
void xen_hvm_inject_msi(uint64_t addr, uint32_t data)
{
161
    xen_xc_hvm_inject_msi(xen_xc, xen_domid, addr, data);
162 163
}

164
static void xen_suspend_notifier(Notifier *notifier, void *data)
165
{
166
    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
167 168
}

169 170 171 172 173 174 175 176 177 178 179 180
/* Xen Interrupt Controller */

static void xen_set_irq(void *opaque, int irq, int level)
{
    xc_hvm_set_isa_irq_level(xen_xc, xen_domid, irq, level);
}

qemu_irq *xen_interrupt_controller_init(void)
{
    return qemu_allocate_irqs(xen_set_irq, NULL, 16);
}

181 182
/* Memory Ops */

183 184 185
static void xen_ram_init(ram_addr_t *below_4g_mem_size,
                         ram_addr_t *above_4g_mem_size,
                         ram_addr_t ram_size, MemoryRegion **ram_memory_p)
186
{
Avi Kivity's avatar
Avi Kivity committed
187 188
    MemoryRegion *sysmem = get_system_memory();
    ram_addr_t block_len;
189 190 191
    uint64_t user_lowmem = object_property_get_int(qdev_get_machine(),
                                                   PC_MACHINE_MAX_RAM_BELOW_4G,
                                                   &error_abort);
192

193
    /* Handle the machine opt max-ram-below-4g.  It is basically doing
194 195 196 197
     * min(xen limit, user limit).
     */
    if (HVM_BELOW_4G_RAM_END <= user_lowmem) {
        user_lowmem = HVM_BELOW_4G_RAM_END;
198
    }
199

200 201 202
    if (ram_size >= user_lowmem) {
        *above_4g_mem_size = ram_size - user_lowmem;
        *below_4g_mem_size = user_lowmem;
203
    } else {
204 205
        *above_4g_mem_size = 0;
        *below_4g_mem_size = ram_size;
206
    }
207 208 209 210 211 212 213 214 215
    if (!*above_4g_mem_size) {
        block_len = ram_size;
    } else {
        /*
         * Xen does not allocate the memory continuously, it keeps a
         * hole of the size computed above or passed in.
         */
        block_len = (1ULL << 32) + *above_4g_mem_size;
    }
216 217
    memory_region_init_ram(&ram_memory, NULL, "xen.ram", block_len,
                           &error_abort);
218 219
    *ram_memory_p = &ram_memory;
    vmstate_register_ram_global(&ram_memory);
220

221
    memory_region_init_alias(&ram_640k, NULL, "xen.ram.640k",
Avi Kivity's avatar
Avi Kivity committed
222 223
                             &ram_memory, 0, 0xa0000);
    memory_region_add_subregion(sysmem, 0, &ram_640k);
224 225 226 227 228 229
    /* Skip of the VGA IO memory space, it will be registered later by the VGA
     * emulated device.
     *
     * The area between 0xc0000 and 0x100000 will be used by SeaBIOS to load
     * the Options ROM, so it is registered here as RAM.
     */
230
    memory_region_init_alias(&ram_lo, NULL, "xen.ram.lo",
231 232
                             &ram_memory, 0xc0000,
                             *below_4g_mem_size - 0xc0000);
Avi Kivity's avatar
Avi Kivity committed
233
    memory_region_add_subregion(sysmem, 0xc0000, &ram_lo);
234
    if (*above_4g_mem_size > 0) {
235
        memory_region_init_alias(&ram_hi, NULL, "xen.ram.hi",
Avi Kivity's avatar
Avi Kivity committed
236
                                 &ram_memory, 0x100000000ULL,
237
                                 *above_4g_mem_size);
Avi Kivity's avatar
Avi Kivity committed
238
        memory_region_add_subregion(sysmem, 0x100000000ULL, &ram_hi);
239 240 241
    }
}

242
void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr)
243 244 245 246 247
{
    unsigned long nr_pfn;
    xen_pfn_t *pfn_list;
    int i;

248 249 250 251 252 253 254 255
    if (runstate_check(RUN_STATE_INMIGRATE)) {
        /* RAM already populated in Xen */
        fprintf(stderr, "%s: do not alloc "RAM_ADDR_FMT
                " bytes of ram at "RAM_ADDR_FMT" when runstate is INMIGRATE\n",
                __func__, size, ram_addr); 
        return;
    }

Avi Kivity's avatar
Avi Kivity committed
256 257 258 259
    if (mr == &ram_memory) {
        return;
    }

260 261 262
    trace_xen_ram_alloc(ram_addr, size);

    nr_pfn = size >> TARGET_PAGE_BITS;
263
    pfn_list = g_malloc(sizeof (*pfn_list) * nr_pfn);
264 265 266 267 268 269

    for (i = 0; i < nr_pfn; i++) {
        pfn_list[i] = (ram_addr >> TARGET_PAGE_BITS) + i;
    }

    if (xc_domain_populate_physmap_exact(xen_xc, xen_domid, nr_pfn, 0, 0, pfn_list)) {
270
        hw_error("xen: failed to populate ram at " RAM_ADDR_FMT, ram_addr);
271 272
    }

273
    g_free(pfn_list);
274 275
}

276
static XenPhysmap *get_physmapping(XenIOState *state,
277
                                   hwaddr start_addr, ram_addr_t size)
278 279 280 281 282 283 284 285 286 287 288 289 290
{
    XenPhysmap *physmap = NULL;

    start_addr &= TARGET_PAGE_MASK;

    QLIST_FOREACH(physmap, &state->physmap, list) {
        if (range_covers_byte(physmap->start_addr, physmap->size, start_addr)) {
            return physmap;
        }
    }
    return NULL;
}

291
static hwaddr xen_phys_offset_to_gaddr(hwaddr start_addr,
292 293
                                                   ram_addr_t size, void *opaque)
{
294
    hwaddr addr = start_addr & TARGET_PAGE_MASK;
295 296 297 298 299 300 301 302 303 304 305 306
    XenIOState *xen_io_state = opaque;
    XenPhysmap *physmap = NULL;

    QLIST_FOREACH(physmap, &xen_io_state->physmap, list) {
        if (range_covers_byte(physmap->phys_offset, physmap->size, addr)) {
            return physmap->start_addr;
        }
    }

    return start_addr;
}

307 308
#if CONFIG_XEN_CTRL_INTERFACE_VERSION >= 340
static int xen_add_to_physmap(XenIOState *state,
309
                              hwaddr start_addr,
310
                              ram_addr_t size,
311
                              MemoryRegion *mr,
312
                              hwaddr offset_within_region)
313 314 315 316
{
    unsigned long i = 0;
    int rc = 0;
    XenPhysmap *physmap = NULL;
317 318
    hwaddr pfn, start_gpfn;
    hwaddr phys_offset = memory_region_get_ram_addr(mr);
319
    char path[80], value[17];
320
    const char *mr_name;
321 322 323 324 325 326 327 328

    if (get_physmapping(state, start_addr, size)) {
        return 0;
    }
    if (size <= 0) {
        return -1;
    }

329 330 331 332
    /* Xen can only handle a single dirty log region for now and we want
     * the linear framebuffer to be that region.
     * Avoid tracking any regions that is not videoram and avoid tracking
     * the legacy vga region. */
333 334
    if (mr == framebuffer && start_addr > 0xbffff) {
        goto go_physmap;
335 336 337 338
    }
    return -1;

go_physmap:
339 340
    DPRINTF("mapping vram to %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
            start_addr, start_addr + size);
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355

    pfn = phys_offset >> TARGET_PAGE_BITS;
    start_gpfn = start_addr >> TARGET_PAGE_BITS;
    for (i = 0; i < size >> TARGET_PAGE_BITS; i++) {
        unsigned long idx = pfn + i;
        xen_pfn_t gpfn = start_gpfn + i;

        rc = xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
        if (rc) {
            DPRINTF("add_to_physmap MFN %"PRI_xen_pfn" to PFN %"
                    PRI_xen_pfn" failed: %d\n", idx, gpfn, rc);
            return -rc;
        }
    }

356 357
    mr_name = memory_region_name(mr);

358
    physmap = g_malloc(sizeof (XenPhysmap));
359 360 361

    physmap->start_addr = start_addr;
    physmap->size = size;
362
    physmap->name = mr_name;
363 364 365 366 367 368
    physmap->phys_offset = phys_offset;

    QLIST_INSERT_HEAD(&state->physmap, physmap, list);

    xc_domain_pin_memory_cacheattr(xen_xc, xen_domid,
                                   start_addr >> TARGET_PAGE_BITS,
369
                                   (start_addr + size - 1) >> TARGET_PAGE_BITS,
370
                                   XEN_DOMCTL_MEM_CACHEATTR_WB);
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385

    snprintf(path, sizeof(path),
            "/local/domain/0/device-model/%d/physmap/%"PRIx64"/start_addr",
            xen_domid, (uint64_t)phys_offset);
    snprintf(value, sizeof(value), "%"PRIx64, (uint64_t)start_addr);
    if (!xs_write(state->xenstore, 0, path, value, strlen(value))) {
        return -1;
    }
    snprintf(path, sizeof(path),
            "/local/domain/0/device-model/%d/physmap/%"PRIx64"/size",
            xen_domid, (uint64_t)phys_offset);
    snprintf(value, sizeof(value), "%"PRIx64, (uint64_t)size);
    if (!xs_write(state->xenstore, 0, path, value, strlen(value))) {
        return -1;
    }
386
    if (mr_name) {
387 388 389
        snprintf(path, sizeof(path),
                "/local/domain/0/device-model/%d/physmap/%"PRIx64"/name",
                xen_domid, (uint64_t)phys_offset);
390
        if (!xs_write(state->xenstore, 0, path, mr_name, strlen(mr_name))) {
391 392 393 394
            return -1;
        }
    }

395 396 397 398
    return 0;
}

static int xen_remove_from_physmap(XenIOState *state,
399
                                   hwaddr start_addr,
400 401 402 403 404
                                   ram_addr_t size)
{
    unsigned long i = 0;
    int rc = 0;
    XenPhysmap *physmap = NULL;
405
    hwaddr phys_offset = 0;
406 407 408 409 410 411 412 413 414

    physmap = get_physmapping(state, start_addr, size);
    if (physmap == NULL) {
        return -1;
    }

    phys_offset = physmap->phys_offset;
    size = physmap->size;

415 416
    DPRINTF("unmapping vram to %"HWADDR_PRIx" - %"HWADDR_PRIx", at "
            "%"HWADDR_PRIx"\n", start_addr, start_addr + size, phys_offset);
417 418 419 420 421

    size >>= TARGET_PAGE_BITS;
    start_addr >>= TARGET_PAGE_BITS;
    phys_offset >>= TARGET_PAGE_BITS;
    for (i = 0; i < size; i++) {
Stefano Stabellini's avatar
Stefano Stabellini committed
422
        xen_pfn_t idx = start_addr + i;
423 424 425 426 427 428 429 430 431 432 433 434 435 436
        xen_pfn_t gpfn = phys_offset + i;

        rc = xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
        if (rc) {
            fprintf(stderr, "add_to_physmap MFN %"PRI_xen_pfn" to PFN %"
                    PRI_xen_pfn" failed: %d\n", idx, gpfn, rc);
            return -rc;
        }
    }

    QLIST_REMOVE(physmap, list);
    if (state->log_for_dirtybit == physmap) {
        state->log_for_dirtybit = NULL;
    }
437
    g_free(physmap);
438 439 440 441 442 443

    return 0;
}

#else
static int xen_add_to_physmap(XenIOState *state,
444
                              hwaddr start_addr,
445
                              ram_addr_t size,
446
                              MemoryRegion *mr,
447
                              hwaddr offset_within_region)
448 449 450 451 452
{
    return -ENOSYS;
}

static int xen_remove_from_physmap(XenIOState *state,
453
                                   hwaddr start_addr,
454 455 456 457 458 459
                                   ram_addr_t size)
{
    return -ENOSYS;
}
#endif

460 461 462
static void xen_set_memory(struct MemoryListener *listener,
                           MemoryRegionSection *section,
                           bool add)
463
{
464
    XenIOState *state = container_of(listener, XenIOState, memory_listener);
465
    hwaddr start_addr = section->offset_within_address_space;
466
    ram_addr_t size = int128_get64(section->size);
467
    bool log_dirty = memory_region_is_logging(section->mr);
468 469
    hvmmem_type_t mem_type;

470
    if (!memory_region_is_ram(section->mr)) {
471 472 473
        return;
    }

474 475 476 477 478 479
    if (!(section->mr != &ram_memory
          && ( (log_dirty && add) || (!log_dirty && !add)))) {
        return;
    }

    trace_xen_client_set_memory(start_addr, size, log_dirty);
480 481 482

    start_addr &= TARGET_PAGE_MASK;
    size = TARGET_PAGE_ALIGN(size);
483 484 485 486 487 488 489 490 491 492 493 494 495

    if (add) {
        if (!memory_region_is_rom(section->mr)) {
            xen_add_to_physmap(state, start_addr, size,
                               section->mr, section->offset_within_region);
        } else {
            mem_type = HVMMEM_ram_ro;
            if (xc_hvm_set_mem_type(xen_xc, xen_domid, mem_type,
                                    start_addr >> TARGET_PAGE_BITS,
                                    size >> TARGET_PAGE_BITS)) {
                DPRINTF("xc_hvm_set_mem_type error, addr: "TARGET_FMT_plx"\n",
                        start_addr);
            }
496
        }
497
    } else {
498 499 500 501 502 503
        if (xen_remove_from_physmap(state, start_addr, size) < 0) {
            DPRINTF("physmapping does not exist at "TARGET_FMT_plx"\n", start_addr);
        }
    }
}

504 505 506
static void xen_region_add(MemoryListener *listener,
                           MemoryRegionSection *section)
{
507
    memory_region_ref(section->mr);
508 509 510 511 512 513 514
    xen_set_memory(listener, section, true);
}

static void xen_region_del(MemoryListener *listener,
                           MemoryRegionSection *section)
{
    xen_set_memory(listener, section, false);
515
    memory_region_unref(section->mr);
516 517
}

518
static void xen_sync_dirty_bitmap(XenIOState *state,
519
                                  hwaddr start_addr,
520
                                  ram_addr_t size)
521
{
522
    hwaddr npages = size >> TARGET_PAGE_BITS;
523 524 525 526 527 528 529 530
    const int width = sizeof(unsigned long) * 8;
    unsigned long bitmap[(npages + width - 1) / width];
    int rc, i, j;
    const XenPhysmap *physmap = NULL;

    physmap = get_physmapping(state, start_addr, size);
    if (physmap == NULL) {
        /* not handled */
531
        return;
532 533 534 535 536
    }

    if (state->log_for_dirtybit == NULL) {
        state->log_for_dirtybit = physmap;
    } else if (state->log_for_dirtybit != physmap) {
537 538
        /* Only one range for dirty bitmap can be tracked. */
        return;
539 540 541 542 543
    }

    rc = xc_hvm_track_dirty_vram(xen_xc, xen_domid,
                                 start_addr >> TARGET_PAGE_BITS, npages,
                                 bitmap);
544
    if (rc < 0) {
545 546 547 548
#ifndef ENODATA
#define ENODATA  ENOENT
#endif
        if (errno == ENODATA) {
549 550
            memory_region_set_dirty(framebuffer, 0, size);
            DPRINTF("xen: track_dirty_vram failed (0x" TARGET_FMT_plx
551
                    ", 0x" TARGET_FMT_plx "): %s\n",
552
                    start_addr, start_addr + size, strerror(errno));
553 554
        }
        return;
555 556 557 558 559
    }

    for (i = 0; i < ARRAY_SIZE(bitmap); i++) {
        unsigned long map = bitmap[i];
        while (map != 0) {
560
            j = ctzl(map);
561
            map &= ~(1ul << j);
562
            memory_region_set_dirty(framebuffer,
563 564
                                    (i * width + j) * TARGET_PAGE_SIZE,
                                    TARGET_PAGE_SIZE);
565 566 567 568
        };
    }
}

569 570
static void xen_log_start(MemoryListener *listener,
                          MemoryRegionSection *section)
571
{
572
    XenIOState *state = container_of(listener, XenIOState, memory_listener);
573

574
    xen_sync_dirty_bitmap(state, section->offset_within_address_space,
575
                          int128_get64(section->size));
576 577
}

578
static void xen_log_stop(MemoryListener *listener, MemoryRegionSection *section)
579
{
580
    XenIOState *state = container_of(listener, XenIOState, memory_listener);
581 582 583

    state->log_for_dirtybit = NULL;
    /* Disable dirty bit tracking */
584
    xc_hvm_track_dirty_vram(xen_xc, xen_domid, 0, 0, NULL);
585 586
}

587
static void xen_log_sync(MemoryListener *listener, MemoryRegionSection *section)
588
{
589
    XenIOState *state = container_of(listener, XenIOState, memory_listener);
590

591
    xen_sync_dirty_bitmap(state, section->offset_within_address_space,
592
                          int128_get64(section->size));
593 594
}

595 596
static void xen_log_global_start(MemoryListener *listener)
{
597 598 599
    if (xen_enabled()) {
        xen_in_migration = true;
    }
600 601 602
}

static void xen_log_global_stop(MemoryListener *listener)
603
{
604
    xen_in_migration = false;
605 606
}

607 608 609
static MemoryListener xen_memory_listener = {
    .region_add = xen_region_add,
    .region_del = xen_region_del,
610 611
    .log_start = xen_log_start,
    .log_stop = xen_log_stop,
612 613 614
    .log_sync = xen_log_sync,
    .log_global_start = xen_log_global_start,
    .log_global_stop = xen_log_global_stop,
615
    .priority = 10,
616
};
617

618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
/* get the ioreq packets from share mem */
static ioreq_t *cpu_get_ioreq_from_shared_memory(XenIOState *state, int vcpu)
{
    ioreq_t *req = xen_vcpu_ioreq(state->shared_page, vcpu);

    if (req->state != STATE_IOREQ_READY) {
        DPRINTF("I/O request not ready: "
                "%x, ptr: %x, port: %"PRIx64", "
                "data: %"PRIx64", count: %" FMT_ioreq_size ", size: %" FMT_ioreq_size "\n",
                req->state, req->data_is_ptr, req->addr,
                req->data, req->count, req->size);
        return NULL;
    }

    xen_rmb(); /* see IOREQ_READY /then/ read contents of ioreq */

    req->state = STATE_IOREQ_INPROCESS;
    return req;
}

/* use poll to get the port notification */
/* ioreq_vec--out,the */
/* retval--the number of ioreq packet */
static ioreq_t *cpu_get_ioreq(XenIOState *state)
{
    int i;
    evtchn_port_t port;

    port = xc_evtchn_pending(state->xce_handle);
647
    if (port == state->bufioreq_local_port) {
648 649
        timer_mod(state->buffered_io_timer,
                BUFFER_IO_MAX_DELAY + qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
650 651 652
        return NULL;
    }

653
    if (port != -1) {
654
        for (i = 0; i < max_cpus; i++) {
655 656 657 658 659
            if (state->ioreq_local_port[i] == port) {
                break;
            }
        }

660
        if (i == max_cpus) {
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
            hw_error("Fatal error while trying to get io event!\n");
        }

        /* unmask the wanted port again */
        xc_evtchn_unmask(state->xce_handle, port);

        /* get the io packet from shared memory */
        state->send_vcpu = i;
        return cpu_get_ioreq_from_shared_memory(state, i);
    }

    /* read error or read nothing */
    return NULL;
}

static uint32_t do_inp(pio_addr_t addr, unsigned long size)
{
    switch (size) {
        case 1:
            return cpu_inb(addr);
        case 2:
            return cpu_inw(addr);
        case 4:
            return cpu_inl(addr);
        default:
            hw_error("inp: bad size: %04"FMT_pioaddr" %lx", addr, size);
    }
}

static void do_outp(pio_addr_t addr,
        unsigned long size, uint32_t val)
{
    switch (size) {
        case 1:
            return cpu_outb(addr, val);
        case 2:
            return cpu_outw(addr, val);
        case 4:
            return cpu_outl(addr, val);
        default:
            hw_error("outp: bad size: %04"FMT_pioaddr" %lx", addr, size);
    }
}

705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730
/*
 * Helper functions which read/write an object from/to physical guest
 * memory, as part of the implementation of an ioreq.
 *
 * Equivalent to
 *   cpu_physical_memory_rw(addr + (req->df ? -1 : +1) * req->size * i,
 *                          val, req->size, 0/1)
 * except without the integer overflow problems.
 */
static void rw_phys_req_item(hwaddr addr,
                             ioreq_t *req, uint32_t i, void *val, int rw)
{
    /* Do everything unsigned so overflow just results in a truncated result
     * and accesses to undesired parts of guest memory, which is up
     * to the guest */
    hwaddr offset = (hwaddr)req->size * i;
    if (req->df) {
        addr -= offset;
    } else {
        addr += offset;
    }
    cpu_physical_memory_rw(addr, val, req->size, rw);
}

static inline void read_phys_req_item(hwaddr addr,
                                      ioreq_t *req, uint32_t i, void *val)
731
{
732 733 734 735 736 737 738
    rw_phys_req_item(addr, req, i, val, 0);
}
static inline void write_phys_req_item(hwaddr addr,
                                       ioreq_t *req, uint32_t i, void *val)
{
    rw_phys_req_item(addr, req, i, val, 1);
}
739

740 741 742

static void cpu_ioreq_pio(ioreq_t *req)
{
743
    uint32_t i;
744 745 746 747 748 749 750 751 752

    if (req->dir == IOREQ_READ) {
        if (!req->data_is_ptr) {
            req->data = do_inp(req->addr, req->size);
        } else {
            uint32_t tmp;

            for (i = 0; i < req->count; i++) {
                tmp = do_inp(req->addr, req->size);
753
                write_phys_req_item(req->data, req, i, &tmp);
754 755 756 757 758 759 760 761 762
            }
        }
    } else if (req->dir == IOREQ_WRITE) {
        if (!req->data_is_ptr) {
            do_outp(req->addr, req->size, req->data);
        } else {
            for (i = 0; i < req->count; i++) {
                uint32_t tmp = 0;

763
                read_phys_req_item(req->data, req, i, &tmp);
764 765 766 767 768 769 770 771
                do_outp(req->addr, req->size, tmp);
            }
        }
    }
}

static void cpu_ioreq_move(ioreq_t *req)
{
772
    uint32_t i;
773 774 775 776

    if (!req->data_is_ptr) {
        if (req->dir == IOREQ_READ) {
            for (i = 0; i < req->count; i++) {
777
                read_phys_req_item(req->addr, req, i, &req->data);
778 779 780
            }
        } else if (req->dir == IOREQ_WRITE) {
            for (i = 0; i < req->count; i++) {
781
                write_phys_req_item(req->addr, req, i, &req->data);
782 783 784
            }
        }
    } else {
785
        uint64_t tmp;
786 787 788

        if (req->dir == IOREQ_READ) {
            for (i = 0; i < req->count; i++) {
789 790
                read_phys_req_item(req->addr, req, i, &tmp);
                write_phys_req_item(req->data, req, i, &tmp);
791 792 793
            }
        } else if (req->dir == IOREQ_WRITE) {
            for (i = 0; i < req->count; i++) {
794 795
                read_phys_req_item(req->data, req, i, &tmp);
                write_phys_req_item(req->addr, req, i, &tmp);
796 797 798 799 800
            }
        }
    }
}

801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844
static void regs_to_cpu(vmware_regs_t *vmport_regs, ioreq_t *req)
{
    X86CPU *cpu;
    CPUX86State *env;

    cpu = X86_CPU(current_cpu);
    env = &cpu->env;
    env->regs[R_EAX] = req->data;
    env->regs[R_EBX] = vmport_regs->ebx;
    env->regs[R_ECX] = vmport_regs->ecx;
    env->regs[R_EDX] = vmport_regs->edx;
    env->regs[R_ESI] = vmport_regs->esi;
    env->regs[R_EDI] = vmport_regs->edi;
}

static void regs_from_cpu(vmware_regs_t *vmport_regs)
{
    X86CPU *cpu = X86_CPU(current_cpu);
    CPUX86State *env = &cpu->env;

    vmport_regs->ebx = env->regs[R_EBX];
    vmport_regs->ecx = env->regs[R_ECX];
    vmport_regs->edx = env->regs[R_EDX];
    vmport_regs->esi = env->regs[R_ESI];
    vmport_regs->edi = env->regs[R_EDI];
}

static void handle_vmport_ioreq(XenIOState *state, ioreq_t *req)
{
    vmware_regs_t *vmport_regs;

    assert(state->shared_vmport_page);
    vmport_regs =
        &state->shared_vmport_page->vcpu_vmport_regs[state->send_vcpu];
    QEMU_BUILD_BUG_ON(sizeof(*req) < sizeof(*vmport_regs));

    current_cpu = state->cpu_by_vcpu_id[state->send_vcpu];
    regs_to_cpu(vmport_regs, req);
    cpu_ioreq_pio(req);
    regs_from_cpu(vmport_regs);
    current_cpu = NULL;
}

static void handle_ioreq(XenIOState *state, ioreq_t *req)
845 846 847 848 849 850 851 852 853 854 855 856 857
{
    if (!req->data_is_ptr && (req->dir == IOREQ_WRITE) &&
            (req->size < sizeof (target_ulong))) {
        req->data &= ((target_ulong) 1 << (8 * req->size)) - 1;
    }

    switch (req->type) {
        case IOREQ_TYPE_PIO:
            cpu_ioreq_pio(req);
            break;
        case IOREQ_TYPE_COPY:
            cpu_ioreq_move(req);
            break;
858 859 860
        case IOREQ_TYPE_VMWARE_PORT:
            handle_vmport_ioreq(state, req);
            break;
861 862 863
        case IOREQ_TYPE_TIMEOFFSET:
            break;
        case IOREQ_TYPE_INVALIDATE:
864
            xen_invalidate_map_cache();
865 866 867 868 869 870
            break;
        default:
            hw_error("Invalid ioreq type 0x%x\n", req->type);
    }
}

871
static int handle_buffered_iopage(XenIOState *state)
872 873 874 875 876 877
{
    buf_ioreq_t *buf_req = NULL;
    ioreq_t req;
    int qw;

    if (!state->buffered_io_page) {
878
        return 0;
879 880
    }

881 882
    memset(&req, 0x00, sizeof(req));

883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901
    while (state->buffered_io_page->read_pointer != state->buffered_io_page->write_pointer) {
        buf_req = &state->buffered_io_page->buf_ioreq[
            state->buffered_io_page->read_pointer % IOREQ_BUFFER_SLOT_NUM];
        req.size = 1UL << buf_req->size;
        req.count = 1;
        req.addr = buf_req->addr;
        req.data = buf_req->data;
        req.state = STATE_IOREQ_READY;
        req.dir = buf_req->dir;
        req.df = 1;
        req.type = buf_req->type;
        req.data_is_ptr = 0;
        qw = (req.size == 8);
        if (qw) {
            buf_req = &state->buffered_io_page->buf_ioreq[
                (state->buffered_io_page->read_pointer + 1) % IOREQ_BUFFER_SLOT_NUM];
            req.data |= ((uint64_t)buf_req->data) << 32;
        }

902
        handle_ioreq(state, &req);
903 904 905 906

        xen_mb();
        state->buffered_io_page->read_pointer += qw ? 2 : 1;
    }
907 908

    return req.count;
909 910 911 912 913 914
}

static void handle_buffered_io(void *opaque)
{
    XenIOState *state = opaque;

915
    if (handle_buffered_iopage(state)) {
916 917
        timer_mod(state->buffered_io_timer,
                BUFFER_IO_MAX_DELAY + qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
918
    } else {
919
        timer_del(state->buffered_io_timer);
920 921
        xc_evtchn_unmask(state->xce_handle, state->bufioreq_local_port);
    }
922 923 924 925 926 927 928 929 930
}

static void cpu_handle_ioreq(void *opaque)
{
    XenIOState *state = opaque;
    ioreq_t *req = cpu_get_ioreq(state);

    handle_buffered_iopage(state);
    if (req) {
931
        handle_ioreq(state, req);
932 933 934 935

        if (req->state != STATE_IOREQ_INPROCESS) {
            fprintf(stderr, "Badness in I/O request ... not in service?!: "
                    "%x, ptr: %x, port: %"PRIx64", "
936 937 938
                    "data: %"PRIx64", count: %" FMT_ioreq_size
                    ", size: %" FMT_ioreq_size
                    ", type: %"FMT_ioreq_size"\n",
939
                    req->state, req->data_is_ptr, req->addr,
940
                    req->data, req->count, req->size, req->type);
941
            destroy_hvm_domain(false);
942 943 944 945 946 947 948 949 950 951 952
            return;
        }

        xen_wmb(); /* Update ioreq contents /then/ update state. */

        /*
         * We do this before we send the response so that the tools
         * have the opportunity to pick up on the reset before the
         * guest resumes and does a hlt with interrupts disabled which
         * causes Xen to powerdown the domain.
         */
953
        if (runstate_is_running()) {
954
            if (qemu_shutdown_requested_get()) {
955
                destroy_hvm_domain(false);
956 957
            }
            if (qemu_reset_requested_get()) {
Jan Kiszka's avatar
Jan Kiszka committed
958
                qemu_system_reset(VMRESET_REPORT);
959
                destroy_hvm_domain(true);
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975
            }
        }

        req->state = STATE_IORESP_READY;
        xc_evtchn_notify(state->xce_handle, state->ioreq_local_port[state->send_vcpu]);
    }
}

static void xen_main_loop_prepare(XenIOState *state)
{
    int evtchn_fd = -1;

    if (state->xce_handle != XC_HANDLER_INITIAL_VALUE) {
        evtchn_fd = xc_evtchn_fd(state->xce_handle);
    }

976
    state->buffered_io_timer = timer_new_ms(QEMU_CLOCK_REALTIME, handle_buffered_io,
977 978 979
                                                 state);

    if (evtchn_fd != -1) {
980 981 982 983 984 985 986 987
        CPUState *cpu_state;

        DPRINTF("%s: Init cpu_by_vcpu_id\n", __func__);
        CPU_FOREACH(cpu_state) {
            DPRINTF("%s: cpu_by_vcpu_id[%d]=%p\n",
                    __func__, cpu_state->cpu_index, cpu_state);
            state->cpu_by_vcpu_id[cpu_state->cpu_index] = cpu_state;
        }
988 989 990 991 992
        qemu_set_fd_handler(evtchn_fd, cpu_handle_ioreq, NULL, state);
    }
}


993 994
static void xen_hvm_change_state_handler(void *opaque, int running,
                                         RunState rstate)
995 996
{
    if (running) {
997
        xen_main_loop_prepare((XenIOState *)opaque);
998 999 1000
    }
}

1001
static void xen_exit_notifier(Notifier *n, void *data)
1002 1003 1004 1005
{
    XenIOState *state = container_of(n, XenIOState, exit);

    xc_evtchn_close(state->xce_handle);
1006
    xs_daemon_close(state->xenstore);
1007 1008
}

1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
static void xen_read_physmap(XenIOState *state)
{
    XenPhysmap *physmap = NULL;
    unsigned int len, num, i;
    char path[80], *value = NULL;
    char **entries = NULL;

    snprintf(path, sizeof(path),
            "/local/domain/0/device-model/%d/physmap", xen_domid);
    entries = xs_directory(state->xenstore, 0, path, &num);
    if (entries == NULL)
        return;

    for (i = 0; i < num; i++) {
        physmap = g_malloc(sizeof (XenPhysmap));
        physmap->phys_offset = strtoull(entries[i], NULL, 16);
        snprintf(path, sizeof(path),
                "/local/domain/0/device-model/%d/physmap/%s/start_addr",
                xen_domid, entries[i]);
        value = xs_read(state->xenstore, 0, path, &len);
        if (value == NULL) {
1030
            g_free(physmap);
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
            continue;
        }
        physmap->start_addr = strtoull(value, NULL, 16);
        free(value);

        snprintf(path, sizeof(path),
                "/local/domain/0/device-model/%d/physmap/%s/size",
                xen_domid, entries[i]);
        value = xs_read(state->xenstore, 0, path, &len);
        if (value == NULL) {
1041
            g_free(physmap);
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
            continue;
        }
        physmap->size = strtoull(value, NULL, 16);
        free(value);

        snprintf(path, sizeof(path),
                "/local/domain/0/device-model/%d/physmap/%s/name",
                xen_domid, entries[i]);
        physmap->name = xs_read(state->xenstore, 0, path, &len);

        QLIST_INSERT_HEAD(&state->physmap, physmap, list);
    }
    free(entries);
}

1057 1058 1059 1060 1061
static void xen_wakeup_notifier(Notifier *notifier, void *data)
{
    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
}

1062
/* return 0 means OK, or -1 means critical issue -- will exit(1) */
1063 1064
int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
                 MemoryRegion **ram_memory)
Anthony PERARD's avatar
Anthony PERARD committed
1065
{
1066 1067
    int i, rc;
    unsigned long ioreq_pfn;
1068
    unsigned long bufioreq_evtchn;
1069 1070
    XenIOState *state;

1071
    state = g_malloc0(sizeof (XenIOState));
1072 1073 1074 1075

    state->xce_handle = xen_xc_evtchn_open(NULL, 0);
    if (state->xce_handle == XC_HANDLER_INITIAL_VALUE) {
        perror("xen: event channel open");
1076
        return -1;
1077 1078
    }

1079 1080 1081
    state->xenstore = xs_daemon_open();
    if (state->xenstore == NULL) {
        perror("xen: xenstore open");
1082
        return -1;
1083 1084
    }

1085 1086 1087
    state->exit.notify = xen_exit_notifier;
    qemu_add_exit_notifier(&state->exit);

1088 1089 1090
    state->suspend.notify = xen_suspend_notifier;
    qemu_register_suspend_notifier(&state->suspend);

1091 1092 1093
    state->wakeup.notify = xen_wakeup_notifier;
    qemu_register_wakeup_notifier(&state->wakeup);

1094 1095 1096 1097 1098 1099 1100 1101 1102
    xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
    DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
    state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
                                              PROT_READ|PROT_WRITE, ioreq_pfn);
    if (state->shared_page == NULL) {
        hw_error("map shared IO page returned error %d handle=" XC_INTERFACE_FMT,
                 errno, xen_xc);
    }

1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
    rc = xen_get_vmport_regs_pfn(xen_xc, xen_domid, &ioreq_pfn);
    if (!rc) {
        DPRINTF("shared vmport page at pfn %lx\n", ioreq_pfn);
        state->shared_vmport_page =
            xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
                                 PROT_READ|PROT_WRITE, ioreq_pfn);
        if (state->shared_vmport_page == NULL) {
            hw_error("map shared vmport IO page returned error %d handle="
                     XC_INTERFACE_FMT, errno, xen_xc);
        }
    } else if (rc != -ENOSYS) {
        hw_error("get vmport regs pfn returned error %d, rc=%d", errno, rc);
    }

1117 1118 1119 1120 1121 1122 1123 1124
    xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_BUFIOREQ_PFN, &ioreq_pfn);
    DPRINTF("buffered io page at pfn %lx\n", ioreq_pfn);
    state->buffered_io_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
                                                   PROT_READ|PROT_WRITE, ioreq_pfn);
    if (state->buffered_io_page == NULL) {
        hw_error("map buffered IO page returned error %d", errno);
    }

1125 1126 1127
    /* Note: cpus is empty at this point in init */
    state->cpu_by_vcpu_id = g_malloc0(max_cpus * sizeof(CPUState *));

1128
    state->ioreq_local_port = g_malloc0(max_cpus * sizeof (evtchn_port_t));
1129 1130

    /* FIXME: how about if we overflow the page here? */
1131
    for (i = 0; i < max_cpus; i++) {
1132 1133 1134 1135 1136 1137 1138 1139 1140
        rc = xc_evtchn_bind_interdomain(state->xce_handle, xen_domid,
                                        xen_vcpu_eport(state->shared_page, i));
        if (rc == -1) {
            fprintf(stderr, "bind interdomain ioctl error %d\n", errno);
            return -1;
        }
        state->ioreq_local_port[i] = rc;
    }

1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
    rc = xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_BUFIOREQ_EVTCHN,
            &bufioreq_evtchn);
    if (rc < 0) {
        fprintf(stderr, "failed to get HVM_PARAM_BUFIOREQ_EVTCHN\n");
        return -1;
    }
    rc = xc_evtchn_bind_interdomain(state->xce_handle, xen_domid,
            (uint32_t)bufioreq_evtchn);
    if (rc == -1) {
        fprintf(stderr, "bind interdomain ioctl error %d\n", errno);
        return -1;
    }
    state->bufioreq_local_port = rc;

1155
    /* Init RAM management */
1156
    xen_map_cache_init(xen_phys_offset_to_gaddr, state);
1157
    xen_ram_init(below_4g_mem_size, above_4g_mem_size, ram_size, ram_memory);
1158

1159
    qemu_add_vm_change_state_handler(xen_hvm_change_state_handler, state);
1160

1161
    state->memory_listener = xen_memory_listener;
1162
    QLIST_INIT(&state->physmap);
1163
    memory_listener_register(&state->memory_listener, &address_space_memory);
1164 1165
    state->log_for_dirtybit = NULL;

1166 1167 1168
    /* Initialize backend core & drivers */
    if (xen_be_init() != 0) {
        fprintf(stderr, "%s: xen backend core setup failed\n", __FUNCTION__);
1169
        return -1;
1170 1171
    }
    xen_be_register("console", &xen_console_ops);
1172
    xen_be_register("vkbd", &xen_kbdmouse_ops);
1173
    xen_be_register("qdisk", &xen_blkdev_ops);
1174
    xen_read_physmap(state);
1175

Anthony PERARD's avatar
Anthony PERARD committed
1176 1177
    return 0;
}
1178

1179
void destroy_hvm_domain(bool reboot)
1180 1181 1182 1183 1184 1185 1186 1187
{
    XenXC xc_handle;
    int sts;

    xc_handle = xen_xc_interface_open(0, 0, 0);
    if (xc_handle == XC_HANDLER_INITIAL_VALUE) {
        fprintf(stderr, "Cannot acquire xenctrl handle\n");
    } else {
1188 1189
        sts = xc_domain_shutdown(xc_handle, xen_domid,
                                 reboot ? SHUTDOWN_reboot : SHUTDOWN_poweroff);
1190
        if (sts != 0) {
1191 1192 1193
            fprintf(stderr, "xc_domain_shutdown failed to issue %s, "
                    "sts %d, %s\n", reboot ? "reboot" : "poweroff",
                    sts, strerror(errno));
1194
        } else {
1195 1196
            fprintf(stderr, "Issued domain %d %s\n", xen_domid,
                    reboot ? "reboot" : "poweroff");
1197 1198 1199 1200
        }
        xc_interface_close(xc_handle);
    }
}
1201 1202 1203 1204 1205

void xen_register_framebuffer(MemoryRegion *mr)
{
    framebuffer = mr;
}
1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217

void xen_shutdown_fatal_error(const char *fmt, ...)
{
    va_list ap;

    va_start(ap, fmt);
    vfprintf(stderr, fmt, ap);
    va_end(ap);
    fprintf(stderr, "Will destroy the domain.\n");
    /* destroy the domain */
    qemu_system_shutdown_request();
}
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238

void xen_modified_memory(ram_addr_t start, ram_addr_t length)
{
    if (unlikely(xen_in_migration)) {
        int rc;
        ram_addr_t start_pfn, nb_pages;

        if (length == 0) {
            length = TARGET_PAGE_SIZE;
        }
        start_pfn = start >> TARGET_PAGE_BITS;
        nb_pages = ((start + length + TARGET_PAGE_SIZE - 1) >> TARGET_PAGE_BITS)
            - start_pfn;
        rc = xc_hvm_modified_memory(xen_xc, xen_domid, start_pfn, nb_pages);
        if (rc) {
            fprintf(stderr,
                    "%s failed for "RAM_ADDR_FMT" ("RAM_ADDR_FMT"): %i, %s\n",
                    __func__, start, nb_pages, rc, strerror(-rc));
        }
    }
}
1239 1240 1241 1242 1243 1244 1245 1246 1247

void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
{
    if (enable) {
        memory_global_dirty_log_start();
    } else {
        memory_global_dirty_log_stop();
    }
}