block-migration.c 23.4 KB
Newer Older
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
1 2 3 4 5 6 7 8 9 10 11
/*
 * QEMU live block migration
 *
 * Copyright IBM, Corp. 2009
 *
 * Authors:
 *  Liran Schour   <lirans@il.ibm.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
12 13
 * Contributions after 2012-01-13 are licensed under the terms of the
 * GNU GPL, version 2 or (at your option) any later version.
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
14 15 16
 */

#include "qemu-common.h"
17 18 19
#include "block/block.h"
#include "qemu/error-report.h"
#include "qemu/main-loop.h"
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
20
#include "hw/hw.h"
21 22
#include "qemu/queue.h"
#include "qemu/timer.h"
23 24
#include "migration/block.h"
#include "migration/migration.h"
25
#include "sysemu/blockdev.h"
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
26 27
#include <assert.h>

28 29
#define BLOCK_SIZE                       (1 << 20)
#define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
30 31 32

#define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
#define BLK_MIG_FLAG_EOS                0x02
33
#define BLK_MIG_FLAG_PROGRESS           0x04
34
#define BLK_MIG_FLAG_ZERO_BLOCK         0x08
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
35 36 37 38 39 40

#define MAX_IS_ALLOCATED_SEARCH 65536

//#define DEBUG_BLK_MIGRATION

#ifdef DEBUG_BLK_MIGRATION
malc's avatar
malc committed
41
#define DPRINTF(fmt, ...) \
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
42 43
    do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
#else
malc's avatar
malc committed
44
#define DPRINTF(fmt, ...) \
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
45 46 47
    do { } while (0)
#endif

48
typedef struct BlkMigDevState {
49
    /* Written during setup phase.  Can be read without a lock.  */
50 51 52
    BlockDriverState *bs;
    int shared_base;
    int64_t total_sectors;
53
    QSIMPLEQ_ENTRY(BlkMigDevState) entry;
54 55 56 57 58 59

    /* Only used by migration thread.  Does not need a lock.  */
    int bulk_completed;
    int64_t cur_sector;
    int64_t cur_dirty;

60
    /* Protected by block migration lock.  */
61
    unsigned long *aio_bitmap;
62
    int64_t completed_sectors;
63
    BdrvDirtyBitmap *dirty_bitmap;
64
    Error *blocker;
65 66
} BlkMigDevState;

lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
67
typedef struct BlkMigBlock {
68
    /* Only used by migration thread.  */
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
69 70 71
    uint8_t *buf;
    BlkMigDevState *bmds;
    int64_t sector;
72
    int nr_sectors;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
73 74
    struct iovec iov;
    QEMUIOVector qiov;
75
    BlockAIOCB *aiocb;
76

77
    /* Protected by block migration lock.  */
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
78
    int ret;
79
    QSIMPLEQ_ENTRY(BlkMigBlock) entry;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
80 81 82
} BlkMigBlock;

typedef struct BlkMigState {
83
    /* Written during setup phase.  Can be read without a lock.  */
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
84 85
    int blk_enable;
    int shared_base;
86
    QSIMPLEQ_HEAD(bmds_list, BlkMigDevState) bmds_list;
87
    int64_t total_sector_sum;
88
    bool zero_blocks;
89

90
    /* Protected by lock.  */
91
    QSIMPLEQ_HEAD(blk_list, BlkMigBlock) blk_list;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
92 93
    int submitted;
    int read_done;
94 95

    /* Only used by migration thread.  Does not need a lock.  */
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
96
    int transferred;
97
    int prev_progress;
Liran Schour's avatar
Liran Schour committed
98
    int bulk_completed;
99 100 101

    /* Lock must be taken _inside_ the iothread lock.  */
    QemuMutex lock;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
102 103
} BlkMigState;

104
static BlkMigState block_mig_state;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
105

106 107 108 109 110 111 112 113 114 115
static void blk_mig_lock(void)
{
    qemu_mutex_lock(&block_mig_state.lock);
}

static void blk_mig_unlock(void)
{
    qemu_mutex_unlock(&block_mig_state.lock);
}

116 117 118 119
/* Must run outside of the iothread lock during the bulk phase,
 * or the VM will stall.
 */

120 121 122
static void blk_send(QEMUFile *f, BlkMigBlock * blk)
{
    int len;
123 124 125 126 127 128
    uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;

    if (block_mig_state.zero_blocks &&
        buffer_is_zero(blk->buf, BLOCK_SIZE)) {
        flags |= BLK_MIG_FLAG_ZERO_BLOCK;
    }
129 130 131

    /* sector number and flags */
    qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
132
                     | flags);
133 134

    /* device name */
135
    len = strlen(bdrv_get_device_name(blk->bmds->bs));
136
    qemu_put_byte(f, len);
137
    qemu_put_buffer(f, (uint8_t *)bdrv_get_device_name(blk->bmds->bs), len);
138

139 140 141 142 143 144 145 146
    /* if a block is zero we need to flush here since the network
     * bandwidth is now a lot higher than the storage device bandwidth.
     * thus if we queue zero blocks we slow down the migration */
    if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
        qemu_fflush(f);
        return;
    }

147 148 149
    qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
}

150 151 152 153 154 155 156 157 158 159
int blk_mig_active(void)
{
    return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
}

uint64_t blk_mig_bytes_transferred(void)
{
    BlkMigDevState *bmds;
    uint64_t sum = 0;

160
    blk_mig_lock();
161 162 163
    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
        sum += bmds->completed_sectors;
    }
164
    blk_mig_unlock();
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
    return sum << BDRV_SECTOR_BITS;
}

uint64_t blk_mig_bytes_remaining(void)
{
    return blk_mig_bytes_total() - blk_mig_bytes_transferred();
}

uint64_t blk_mig_bytes_total(void)
{
    BlkMigDevState *bmds;
    uint64_t sum = 0;

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
        sum += bmds->total_sectors;
    }
    return sum << BDRV_SECTOR_BITS;
}

184 185 186

/* Called with migration lock held.  */

187 188 189 190
static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
{
    int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;

191
    if (sector < bdrv_nb_sectors(bmds->bs)) {
192 193 194 195 196 197 198
        return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
            (1UL << (chunk % (sizeof(unsigned long) * 8))));
    } else {
        return 0;
    }
}

199 200
/* Called with migration lock held.  */

201 202 203 204 205 206 207 208 209 210 211 212 213 214
static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
                             int nb_sectors, int set)
{
    int64_t start, end;
    unsigned long val, idx, bit;

    start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
    end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;

    for (; start <= end; start++) {
        idx = start / (sizeof(unsigned long) * 8);
        bit = start % (sizeof(unsigned long) * 8);
        val = bmds->aio_bitmap[idx];
        if (set) {
215
            val |= 1UL << bit;
216
        } else {
217
            val &= ~(1UL << bit);
218 219 220 221 222 223 224 225 226 227
        }
        bmds->aio_bitmap[idx] = val;
    }
}

static void alloc_aio_bitmap(BlkMigDevState *bmds)
{
    BlockDriverState *bs = bmds->bs;
    int64_t bitmap_size;

228
    bitmap_size = bdrv_nb_sectors(bs) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
229 230
    bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;

231
    bmds->aio_bitmap = g_malloc0(bitmap_size);
232 233
}

234 235
/* Never hold migration lock when yielding to the main loop!  */

lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
236 237 238
static void blk_mig_read_cb(void *opaque, int ret)
{
    BlkMigBlock *blk = opaque;
239

240
    blk_mig_lock();
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
241
    blk->ret = ret;
242

243
    QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
244
    bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
245

246 247 248
    block_mig_state.submitted--;
    block_mig_state.read_done++;
    assert(block_mig_state.submitted >= 0);
249
    blk_mig_unlock();
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
250 251
}

252 253
/* Called with no lock taken.  */

254
static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
255
{
256 257 258
    int64_t total_sectors = bmds->total_sectors;
    int64_t cur_sector = bmds->cur_sector;
    BlockDriverState *bs = bmds->bs;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
259
    BlkMigBlock *blk;
260
    int nr_sectors;
261

262
    if (bmds->shared_base) {
263
        qemu_mutex_lock_iothread();
264
        while (cur_sector < total_sectors &&
265 266
               !bdrv_is_allocated(bs, cur_sector, MAX_IS_ALLOCATED_SEARCH,
                                  &nr_sectors)) {
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
267 268
            cur_sector += nr_sectors;
        }
269
        qemu_mutex_unlock_iothread();
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
270
    }
271 272

    if (cur_sector >= total_sectors) {
273
        bmds->cur_sector = bmds->completed_sectors = total_sectors;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
274 275
        return 1;
    }
276

277
    bmds->completed_sectors = cur_sector;
278

279 280
    cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);

281 282
    /* we are going to transfer a full block even if it is not allocated */
    nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
283

284
    if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
285
        nr_sectors = total_sectors - cur_sector;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
286
    }
287

288
    blk = g_new(BlkMigBlock, 1);
289
    blk->buf = g_malloc(BLOCK_SIZE);
290 291
    blk->bmds = bmds;
    blk->sector = cur_sector;
292
    blk->nr_sectors = nr_sectors;
293

Liran Schour's avatar
Liran Schour committed
294 295 296
    blk->iov.iov_base = blk->buf;
    blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
    qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
297

298
    blk_mig_lock();
299
    block_mig_state.submitted++;
300
    blk_mig_unlock();
301

302
    qemu_mutex_lock_iothread();
Liran Schour's avatar
Liran Schour committed
303 304
    blk->aiocb = bdrv_aio_readv(bs, cur_sector, &blk->qiov,
                                nr_sectors, blk_mig_read_cb, blk);
305

306
    bdrv_reset_dirty(bs, cur_sector, nr_sectors);
307
    qemu_mutex_unlock_iothread();
308

309
    bmds->cur_sector = cur_sector + nr_sectors;
310
    return (bmds->cur_sector >= total_sectors);
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
311 312
}

313 314
/* Called with iothread lock taken.  */

315
static int set_dirty_tracking(void)
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
316 317
{
    BlkMigDevState *bmds;
318 319 320 321 322 323 324 325 326 327 328
    int ret;

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
        bmds->dirty_bitmap = bdrv_create_dirty_bitmap(bmds->bs, BLOCK_SIZE,
                                                      NULL);
        if (!bmds->dirty_bitmap) {
            ret = -errno;
            goto fail;
        }
    }
    return 0;
329

330
fail:
331
    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
332 333 334
        if (bmds->dirty_bitmap) {
            bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
        }
335
    }
336
    return ret;
337 338 339 340 341 342 343 344
}

static void unset_dirty_tracking(void)
{
    BlkMigDevState *bmds;

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
        bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
345 346 347
    }
}

348
static void init_blk_migration(QEMUFile *f)
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
349
{
350
    BlockDriverState *bs;
351
    BlkMigDevState *bmds;
352
    int64_t sectors;
353

354 355 356 357 358 359 360 361 362 363 364 365 366
    block_mig_state.submitted = 0;
    block_mig_state.read_done = 0;
    block_mig_state.transferred = 0;
    block_mig_state.total_sector_sum = 0;
    block_mig_state.prev_progress = -1;
    block_mig_state.bulk_completed = 0;
    block_mig_state.zero_blocks = migrate_zero_blocks();

    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
        if (bdrv_is_read_only(bs)) {
            continue;
        }

367
        sectors = bdrv_nb_sectors(bs);
368
        if (sectors <= 0) {
369 370 371
            return;
        }

372
        bmds = g_new0(BlkMigDevState, 1);
373 374 375 376 377
        bmds->bs = bs;
        bmds->bulk_completed = 0;
        bmds->total_sectors = sectors;
        bmds->completed_sectors = 0;
        bmds->shared_base = block_mig_state.shared_base;
378
        alloc_aio_bitmap(bmds);
379 380
        error_setg(&bmds->blocker, "block device is in use by migration");
        bdrv_op_block_all(bs, bmds->blocker);
381
        bdrv_ref(bs);
382 383 384 385

        block_mig_state.total_sector_sum += sectors;

        if (bmds->shared_base) {
386
            DPRINTF("Start migration for %s with shared base image\n",
387
                    bdrv_get_device_name(bs));
388
        } else {
389
            DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
390 391 392 393 394 395
        }

        QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
    }
}

396 397
/* Called with no lock taken.  */

398
static int blk_mig_save_bulked_block(QEMUFile *f)
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
399
{
400
    int64_t completed_sector_sum = 0;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
401
    BlkMigDevState *bmds;
402
    int progress;
403
    int ret = 0;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
404

405
    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
406
        if (bmds->bulk_completed == 0) {
407
            if (mig_save_device_bulk(f, bmds) == 1) {
408 409
                /* completed bulk section for this device */
                bmds->bulk_completed = 1;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
410
            }
411 412 413 414 415
            completed_sector_sum += bmds->completed_sectors;
            ret = 1;
            break;
        } else {
            completed_sector_sum += bmds->completed_sectors;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
416 417
        }
    }
418

419 420 421 422 423 424
    if (block_mig_state.total_sector_sum != 0) {
        progress = completed_sector_sum * 100 /
                   block_mig_state.total_sector_sum;
    } else {
        progress = 100;
    }
425 426 427 428
    if (progress != block_mig_state.prev_progress) {
        block_mig_state.prev_progress = progress;
        qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
                         | BLK_MIG_FLAG_PROGRESS);
429
        DPRINTF("Completed %d %%\r", progress);
430 431 432
    }

    return ret;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
433 434
}

435
static void blk_mig_reset_dirty_cursor(void)
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
436 437
{
    BlkMigDevState *bmds;
438 439 440 441 442 443

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
        bmds->cur_dirty = 0;
    }
}

444 445
/* Called with iothread lock taken.  */

446 447
static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
                                 int is_async)
448 449 450
{
    BlkMigBlock *blk;
    int64_t total_sectors = bmds->total_sectors;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
451
    int64_t sector;
452
    int nr_sectors;
453
    int ret = -EIO;
454

455
    for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
456
        blk_mig_lock();
457
        if (bmds_aio_inflight(bmds, sector)) {
458
            blk_mig_unlock();
459
            bdrv_drain_all();
460 461
        } else {
            blk_mig_unlock();
462
        }
463
        if (bdrv_get_dirty(bmds->bs, bmds->dirty_bitmap, sector)) {
464

465 466 467 468 469
            if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
                nr_sectors = total_sectors - sector;
            } else {
                nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
            }
470
            blk = g_new(BlkMigBlock, 1);
471
            blk->buf = g_malloc(BLOCK_SIZE);
472 473
            blk->bmds = bmds;
            blk->sector = sector;
474
            blk->nr_sectors = nr_sectors;
475

476
            if (is_async) {
477 478 479 480 481 482
                blk->iov.iov_base = blk->buf;
                blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
                qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);

                blk->aiocb = bdrv_aio_readv(bmds->bs, sector, &blk->qiov,
                                            nr_sectors, blk_mig_read_cb, blk);
483 484

                blk_mig_lock();
485
                block_mig_state.submitted++;
486
                bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
487
                blk_mig_unlock();
488
            } else {
489 490
                ret = bdrv_read(bmds->bs, sector, blk->buf, nr_sectors);
                if (ret < 0) {
491
                    goto error;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
492
                }
493
                blk_send(f, blk);
494

495 496
                g_free(blk->buf);
                g_free(blk);
497
            }
498 499 500

            bdrv_reset_dirty(bmds->bs, sector, nr_sectors);
            break;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
501
        }
502 503
        sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
        bmds->cur_dirty = sector;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
504
    }
505

506 507
    return (bmds->cur_dirty >= bmds->total_sectors);

508
error:
509
    DPRINTF("Error reading sector %" PRId64 "\n", sector);
510 511
    g_free(blk->buf);
    g_free(blk);
512
    return ret;
513 514
}

515 516 517
/* Called with iothread lock taken.
 *
 * return value:
518 519 520
 * 0: too much data for max_downtime
 * 1: few enough data for max_downtime
*/
521
static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
522 523
{
    BlkMigDevState *bmds;
524
    int ret = 1;
525 526

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
527
        ret = mig_save_device_dirty(f, bmds, is_async);
528
        if (ret <= 0) {
529 530 531 532 533
            break;
        }
    }

    return ret;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
534 535
}

536 537
/* Called with no locks taken.  */

538
static int flush_blks(QEMUFile *f)
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
539
{
540
    BlkMigBlock *blk;
541
    int ret = 0;
542

malc's avatar
malc committed
543
    DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
544 545
            __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
            block_mig_state.transferred);
546

547
    blk_mig_lock();
548 549 550 551
    while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
        if (qemu_file_rate_limit(f)) {
            break;
        }
552
        if (blk->ret < 0) {
553
            ret = blk->ret;
554 555
            break;
        }
556

557
        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
558
        blk_mig_unlock();
559
        blk_send(f, blk);
560
        blk_mig_lock();
561

562 563
        g_free(blk->buf);
        g_free(blk);
564

565 566 567
        block_mig_state.read_done--;
        block_mig_state.transferred++;
        assert(block_mig_state.read_done >= 0);
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
568
    }
569
    blk_mig_unlock();
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
570

malc's avatar
malc committed
571
    DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
572 573
            block_mig_state.submitted, block_mig_state.read_done,
            block_mig_state.transferred);
574
    return ret;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
575 576
}

577 578
/* Called with iothread lock taken.  */

579 580 581 582 583 584
static int64_t get_remaining_dirty(void)
{
    BlkMigDevState *bmds;
    int64_t dirty = 0;

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
585
        dirty += bdrv_get_dirty_count(bmds->bs, bmds->dirty_bitmap);
586 587
    }

588
    return dirty << BDRV_SECTOR_BITS;
589 590
}

591 592
/* Called with iothread lock taken.  */

593
static void blk_mig_cleanup(void)
594
{
595 596
    BlkMigDevState *bmds;
    BlkMigBlock *blk;
597

598 599
    bdrv_drain_all();

600
    unset_dirty_tracking();
601

602
    blk_mig_lock();
603 604
    while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
605 606
        bdrv_op_unblock_all(bmds->bs, bmds->blocker);
        error_free(bmds->blocker);
607
        bdrv_unref(bmds->bs);
608 609
        g_free(bmds->aio_bitmap);
        g_free(bmds);
610 611
    }

612 613
    while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
614 615
        g_free(blk->buf);
        g_free(blk);
616
    }
617
    blk_mig_unlock();
618 619
}

620 621 622 623 624
static void block_migration_cancel(void *opaque)
{
    blk_mig_cleanup();
}

625
static int block_save_setup(QEMUFile *f, void *opaque)
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
626
{
627 628
    int ret;

629 630
    DPRINTF("Enter save live setup submitted %d transferred %d\n",
            block_mig_state.submitted, block_mig_state.transferred);
631

632
    qemu_mutex_lock_iothread();
633
    init_blk_migration(f);
634 635

    /* start track dirty blocks */
636 637 638 639 640 641 642
    ret = set_dirty_tracking();

    if (ret) {
        qemu_mutex_unlock_iothread();
        return ret;
    }

643
    qemu_mutex_unlock_iothread();
644

645
    ret = flush_blks(f);
646 647 648
    blk_mig_reset_dirty_cursor();
    qemu_put_be64(f, BLK_MIG_FLAG_EOS);

649
    return ret;
650 651
}

652
static int block_save_iterate(QEMUFile *f, void *opaque)
653 654
{
    int ret;
655
    int64_t last_ftell = qemu_ftell(f);
656

657 658
    DPRINTF("Enter save live iterate submitted %d transferred %d\n",
            block_mig_state.submitted, block_mig_state.transferred);
659

660
    ret = flush_blks(f);
661 662
    if (ret) {
        return ret;
663 664
    }

665 666
    blk_mig_reset_dirty_cursor();

667
    /* control the rate of transfer */
668
    blk_mig_lock();
669 670 671
    while ((block_mig_state.submitted +
            block_mig_state.read_done) * BLOCK_SIZE <
           qemu_file_get_rate_limit(f)) {
672
        blk_mig_unlock();
673 674 675 676 677 678
        if (block_mig_state.bulk_completed == 0) {
            /* first finish the bulk phase */
            if (blk_mig_save_bulked_block(f) == 0) {
                /* finished saving bulk on all devices */
                block_mig_state.bulk_completed = 1;
            }
679
            ret = 0;
680
        } else {
681 682 683 684
            /* Always called with iothread lock taken for
             * simplicity, block_save_complete also calls it.
             */
            qemu_mutex_lock_iothread();
685
            ret = blk_mig_save_dirty_block(f, 1);
686
            qemu_mutex_unlock_iothread();
687 688 689 690
        }
        if (ret < 0) {
            return ret;
        }
691
        blk_mig_lock();
692 693 694
        if (ret != 0) {
            /* no more dirty blocks */
            break;
695
        }
696
    }
697
    blk_mig_unlock();
698

699
    ret = flush_blks(f);
700 701
    if (ret) {
        return ret;
702 703
    }

704
    qemu_put_be64(f, BLK_MIG_FLAG_EOS);
705
    return qemu_ftell(f) - last_ftell;
706 707
}

708 709
/* Called with iothread lock taken.  */

710 711 712 713 714 715 716
static int block_save_complete(QEMUFile *f, void *opaque)
{
    int ret;

    DPRINTF("Enter save live complete submitted %d transferred %d\n",
            block_mig_state.submitted, block_mig_state.transferred);

717
    ret = flush_blks(f);
718 719 720
    if (ret) {
        return ret;
    }
721

722
    blk_mig_reset_dirty_cursor();
723

724 725
    /* we know for sure that save bulk is completed and
       all async read completed */
726
    blk_mig_lock();
727
    assert(block_mig_state.submitted == 0);
728
    blk_mig_unlock();
729

730 731
    do {
        ret = blk_mig_save_dirty_block(f, 0);
732 733 734
        if (ret < 0) {
            return ret;
        }
735
    } while (ret == 0);
736

737 738
    /* report completion */
    qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
739

740 741
    DPRINTF("Block migration completed\n");

742 743
    qemu_put_be64(f, BLK_MIG_FLAG_EOS);

744
    blk_mig_cleanup();
745
    return 0;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
746 747
}

748 749
static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
{
750
    /* Estimate pending number of bytes to send */
751 752
    uint64_t pending;

753
    qemu_mutex_lock_iothread();
754
    blk_mig_lock();
755
    pending = get_remaining_dirty() +
756 757 758 759 760 761 762
                       block_mig_state.submitted * BLOCK_SIZE +
                       block_mig_state.read_done * BLOCK_SIZE;

    /* Report at least one block pending during bulk phase */
    if (pending == 0 && !block_mig_state.bulk_completed) {
        pending = BLOCK_SIZE;
    }
763
    blk_mig_unlock();
764
    qemu_mutex_unlock_iothread();
765

766 767
    DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
    return pending;
768 769
}

lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
770 771
static int block_load(QEMUFile *f, void *opaque, int version_id)
{
772
    static int banner_printed;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
773 774 775
    int len, flags;
    char device_name[256];
    int64_t addr;
776
    BlockDriverState *bs, *bs_prev = NULL;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
777
    uint8_t *buf;
778 779
    int64_t total_sectors = 0;
    int nr_sectors;
780
    int ret;
781

lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
782 783
    do {
        addr = qemu_get_be64(f);
784

785 786
        flags = addr & ~BDRV_SECTOR_MASK;
        addr >>= BDRV_SECTOR_BITS;
787 788

        if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
789 790 791 792
            /* get device name */
            len = qemu_get_byte(f);
            qemu_get_buffer(f, (uint8_t *)device_name, len);
            device_name[len] = '\0';
793

lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
794
            bs = bdrv_find(device_name);
795 796 797 798 799
            if (!bs) {
                fprintf(stderr, "Error unknown block device %s\n",
                        device_name);
                return -EINVAL;
            }
800

801 802
            if (bs != bs_prev) {
                bs_prev = bs;
803
                total_sectors = bdrv_nb_sectors(bs);
804
                if (total_sectors <= 0) {
805
                    error_report("Error getting length of block device %s",
806 807 808 809 810 811 812 813 814 815 816
                                 device_name);
                    return -EINVAL;
                }
            }

            if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
                nr_sectors = total_sectors - addr;
            } else {
                nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
            }

817
            if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
818 819
                ret = bdrv_write_zeroes(bs, addr, nr_sectors,
                                        BDRV_REQ_MAY_UNMAP);
820 821 822 823 824 825
            } else {
                buf = g_malloc(BLOCK_SIZE);
                qemu_get_buffer(f, buf, BLOCK_SIZE);
                ret = bdrv_write(bs, addr, buf, nr_sectors);
                g_free(buf);
            }
826

827 828 829
            if (ret < 0) {
                return ret;
            }
830 831 832 833 834 835 836 837
        } else if (flags & BLK_MIG_FLAG_PROGRESS) {
            if (!banner_printed) {
                printf("Receiving block device images\n");
                banner_printed = 1;
            }
            printf("Completed %d %%%c", (int)addr,
                   (addr == 100) ? '\n' : '\r');
            fflush(stdout);
838
        } else if (!(flags & BLK_MIG_FLAG_EOS)) {
839
            fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
840 841
            return -EINVAL;
        }
842 843 844
        ret = qemu_file_get_error(f);
        if (ret != 0) {
            return ret;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
845
        }
846 847
    } while (!(flags & BLK_MIG_FLAG_EOS));

lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
848 849 850
    return 0;
}

851
static void block_set_params(const MigrationParams *params, void *opaque)
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
852
{
853 854
    block_mig_state.blk_enable = params->blk;
    block_mig_state.shared_base = params->shared;
855

lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
856
    /* shared base means that blk_enable = 1 */
857
    block_mig_state.blk_enable |= params->shared;
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
858 859
}

860 861 862 863 864
static bool block_is_active(void *opaque)
{
    return block_mig_state.blk_enable == 1;
}

865
static SaveVMHandlers savevm_block_handlers = {
866
    .set_params = block_set_params,
867
    .save_live_setup = block_save_setup,
868 869
    .save_live_iterate = block_save_iterate,
    .save_live_complete = block_save_complete,
870
    .save_live_pending = block_save_pending,
871
    .load_state = block_load,
872
    .cancel = block_migration_cancel,
873
    .is_active = block_is_active,
874 875
};

lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
876
void blk_mig_init(void)
877
{
878 879
    QSIMPLEQ_INIT(&block_mig_state.bmds_list);
    QSIMPLEQ_INIT(&block_mig_state.blk_list);
880
    qemu_mutex_init(&block_mig_state.lock);
881

882 883
    register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers,
                         &block_mig_state);
lirans@il.ibm.com's avatar
lirans@il.ibm.com committed
884
}