vhost.c 56.6 KB
Newer Older
1 2 3 4 5 6
/* Copyright (C) 2009 Red Hat, Inc.
 * Copyright (C) 2006 Rusty Russell IBM Corporation
 *
 * Author: Michael S. Tsirkin <mst@redhat.com>
 *
 * Inspiration, some code, and most witty comments come from
Rob Landley's avatar
Rob Landley committed
7
 * Documentation/virtual/lguest/lguest.c, by Rusty Russell
8 9 10 11 12 13 14 15
 *
 * This work is licensed under the terms of the GNU GPL, version 2.
 *
 * Generic code for virtio server in host kernel.
 */

#include <linux/eventfd.h>
#include <linux/vhost.h>
16
#include <linux/uio.h>
17
#include <linux/mm.h>
18
#include <linux/mmu_context.h>
19 20 21 22 23
#include <linux/miscdevice.h>
#include <linux/mutex.h>
#include <linux/poll.h>
#include <linux/file.h>
#include <linux/highmem.h>
24
#include <linux/slab.h>
25
#include <linux/vmalloc.h>
26
#include <linux/kthread.h>
27
#include <linux/cgroup.h>
28
#include <linux/module.h>
29
#include <linux/sort.h>
30
#include <linux/interval_tree_generic.h>
31 32 33

#include "vhost.h"

34 35 36 37
static ushort max_mem_regions = 64;
module_param(max_mem_regions, ushort, 0444);
MODULE_PARM_DESC(max_mem_regions,
	"Maximum number of memory regions in memory map. (default: 64)");
Jason Wang's avatar
Jason Wang committed
38 39 40 41
static int max_iotlb_entries = 2048;
module_param(max_iotlb_entries, int, 0444);
MODULE_PARM_DESC(max_iotlb_entries,
	"Maximum number of iotlb entries. (default: 2048)");
42

43 44 45 46
enum {
	VHOST_MEMORY_F_LOG = 0x1,
};

47 48
#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
49

50 51 52 53
INTERVAL_TREE_DEFINE(struct vhost_umem_node,
		     rb, __u64, __subtree_last,
		     START, LAST, , vhost_umem_interval_tree);

54
#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
55
static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
56 57 58 59
{
	vq->user_be = !virtio_legacy_is_little_endian();
}

60 61 62 63 64 65 66 67 68 69
static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq)
{
	vq->user_be = true;
}

static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq)
{
	vq->user_be = false;
}

70 71 72 73 74 75 76 77 78 79 80 81 82 83
static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
{
	struct vhost_vring_state s;

	if (vq->private_data)
		return -EBUSY;

	if (copy_from_user(&s, argp, sizeof(s)))
		return -EFAULT;

	if (s.num != VHOST_VRING_LITTLE_ENDIAN &&
	    s.num != VHOST_VRING_BIG_ENDIAN)
		return -EINVAL;

84 85 86 87
	if (s.num == VHOST_VRING_BIG_ENDIAN)
		vhost_enable_cross_endian_big(vq);
	else
		vhost_enable_cross_endian_little(vq);
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115

	return 0;
}

static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
				   int __user *argp)
{
	struct vhost_vring_state s = {
		.index = idx,
		.num = vq->user_be
	};

	if (copy_to_user(argp, &s, sizeof(s)))
		return -EFAULT;

	return 0;
}

static void vhost_init_is_le(struct vhost_virtqueue *vq)
{
	/* Note for legacy virtio: user_be is initialized at reset time
	 * according to the host endianness. If userspace does not set an
	 * explicit endianness, the default behavior is native endian, as
	 * expected by legacy virtio.
	 */
	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;
}
#else
116
static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
{
}

static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
{
	return -ENOIOCTLCMD;
}

static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
				   int __user *argp)
{
	return -ENOIOCTLCMD;
}

static void vhost_init_is_le(struct vhost_virtqueue *vq)
{
	if (vhost_has_feature(vq, VIRTIO_F_VERSION_1))
		vq->is_le = true;
}
#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */

138 139 140 141 142
static void vhost_reset_is_le(struct vhost_virtqueue *vq)
{
	vq->is_le = virtio_legacy_is_little_endian();
}

Jason Wang's avatar
Jason Wang committed
143 144 145 146 147 148 149 150 151 152 153 154 155
struct vhost_flush_struct {
	struct vhost_work work;
	struct completion wait_event;
};

static void vhost_flush_work(struct vhost_work *work)
{
	struct vhost_flush_struct *s;

	s = container_of(work, struct vhost_flush_struct, work);
	complete(&s->wait_event);
}

156 157 158 159 160
static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
			    poll_table *pt)
{
	struct vhost_poll *poll;

161
	poll = container_of(pt, struct vhost_poll, table);
162 163 164 165 166 167 168
	poll->wqh = wqh;
	add_wait_queue(wqh, &poll->wait);
}

static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
			     void *key)
{
169 170
	struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);

171 172 173
	if (!((unsigned long)key & poll->mask))
		return 0;

174
	vhost_poll_queue(poll);
175 176 177
	return 0;
}

178
void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
179
{
Jason Wang's avatar
Jason Wang committed
180
	clear_bit(VHOST_WORK_QUEUED, &work->flags);
181 182 183
	work->fn = fn;
	init_waitqueue_head(&work->done);
}
184
EXPORT_SYMBOL_GPL(vhost_work_init);
185

186
/* Init poll structure */
187 188
void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
		     unsigned long mask, struct vhost_dev *dev)
189 190 191 192
{
	init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
	init_poll_funcptr(&poll->table, vhost_poll_func);
	poll->mask = mask;
193
	poll->dev = dev;
194
	poll->wqh = NULL;
195

196
	vhost_work_init(&poll->work, fn);
197
}
198
EXPORT_SYMBOL_GPL(vhost_poll_init);
199 200 201

/* Start polling a file. We add ourselves to file's wait queue. The caller must
 * keep a reference to a file until after vhost_poll_stop is called. */
202
int vhost_poll_start(struct vhost_poll *poll, struct file *file)
203 204
{
	unsigned long mask;
205
	int ret = 0;
206

207 208 209
	if (poll->wqh)
		return 0;

210 211 212
	mask = file->f_op->poll(file, &poll->table);
	if (mask)
		vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
213 214 215 216 217 218 219
	if (mask & POLLERR) {
		if (poll->wqh)
			remove_wait_queue(poll->wqh, &poll->wait);
		ret = -EINVAL;
	}

	return ret;
220
}
221
EXPORT_SYMBOL_GPL(vhost_poll_start);
222 223 224 225 226

/* Stop polling a file. After this function returns, it becomes safe to drop the
 * file reference. You must also flush afterwards. */
void vhost_poll_stop(struct vhost_poll *poll)
{
227 228 229 230
	if (poll->wqh) {
		remove_wait_queue(poll->wqh, &poll->wait);
		poll->wqh = NULL;
	}
231
}
232
EXPORT_SYMBOL_GPL(vhost_poll_stop);
233

Jason Wang's avatar
Jason Wang committed
234
void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
235
{
Jason Wang's avatar
Jason Wang committed
236
	struct vhost_flush_struct flush;
237

Jason Wang's avatar
Jason Wang committed
238 239 240
	if (dev->worker) {
		init_completion(&flush.wait_event);
		vhost_work_init(&flush.work, vhost_flush_work);
241

Jason Wang's avatar
Jason Wang committed
242 243 244
		vhost_work_queue(dev, &flush.work);
		wait_for_completion(&flush.wait_event);
	}
245
}
246
EXPORT_SYMBOL_GPL(vhost_work_flush);
247

248 249 250 251 252 253
/* Flush any work that has been scheduled. When calling this, don't hold any
 * locks that are also used by the callback. */
void vhost_poll_flush(struct vhost_poll *poll)
{
	vhost_work_flush(poll->dev, &poll->work);
}
254
EXPORT_SYMBOL_GPL(vhost_poll_flush);
255

256
void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
257
{
Jason Wang's avatar
Jason Wang committed
258 259
	if (!dev->worker)
		return;
260

Jason Wang's avatar
Jason Wang committed
261 262 263 264 265 266
	if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) {
		/* We can only add the work to the list after we're
		 * sure it was not in the list.
		 */
		smp_mb();
		llist_add(&work->node, &dev->work_list);
267 268
		wake_up_process(dev->worker);
	}
269
}
270
EXPORT_SYMBOL_GPL(vhost_work_queue);
271

272 273 274
/* A lockless hint for busy polling code to exit the loop */
bool vhost_has_work(struct vhost_dev *dev)
{
Jason Wang's avatar
Jason Wang committed
275
	return !llist_empty(&dev->work_list);
276 277 278
}
EXPORT_SYMBOL_GPL(vhost_has_work);

279 280 281 282
void vhost_poll_queue(struct vhost_poll *poll)
{
	vhost_work_queue(poll->dev, &poll->work);
}
283
EXPORT_SYMBOL_GPL(vhost_poll_queue);
284

285 286 287 288 289 290 291 292 293 294
static void vhost_vq_reset(struct vhost_dev *dev,
			   struct vhost_virtqueue *vq)
{
	vq->num = 1;
	vq->desc = NULL;
	vq->avail = NULL;
	vq->used = NULL;
	vq->last_avail_idx = 0;
	vq->avail_idx = 0;
	vq->last_used_idx = 0;
295 296
	vq->signalled_used = 0;
	vq->signalled_used_valid = false;
297 298 299 300
	vq->used_flags = 0;
	vq->log_used = false;
	vq->log_addr = -1ull;
	vq->private_data = NULL;
301
	vq->acked_features = 0;
302 303 304 305 306 307
	vq->log_base = NULL;
	vq->error_ctx = NULL;
	vq->error = NULL;
	vq->kick = NULL;
	vq->call_ctx = NULL;
	vq->call = NULL;
308
	vq->log_ctx = NULL;
309 310
	vhost_reset_is_le(vq);
	vhost_disable_cross_endian(vq);
311
	vq->busyloop_timeout = 0;
312
	vq->umem = NULL;
Jason Wang's avatar
Jason Wang committed
313
	vq->iotlb = NULL;
314 315
}

316 317 318
static int vhost_worker(void *data)
{
	struct vhost_dev *dev = data;
Jason Wang's avatar
Jason Wang committed
319 320
	struct vhost_work *work, *work_next;
	struct llist_node *node;
321
	mm_segment_t oldfs = get_fs();
322

323
	set_fs(USER_DS);
324 325
	use_mm(dev->mm);

326 327 328 329 330 331
	for (;;) {
		/* mb paired w/ kthread_stop */
		set_current_state(TASK_INTERRUPTIBLE);

		if (kthread_should_stop()) {
			__set_current_state(TASK_RUNNING);
332
			break;
333
		}
Jason Wang's avatar
Jason Wang committed
334 335 336 337 338 339 340 341 342 343

		node = llist_del_all(&dev->work_list);
		if (!node)
			schedule();

		node = llist_reverse_order(node);
		/* make sure flag is seen after deletion */
		smp_wmb();
		llist_for_each_entry_safe(work, work_next, node, node) {
			clear_bit(VHOST_WORK_QUEUED, &work->flags);
344 345
			__set_current_state(TASK_RUNNING);
			work->fn(work);
346 347
			if (need_resched())
				schedule();
Jason Wang's avatar
Jason Wang committed
348
		}
349
	}
350
	unuse_mm(dev->mm);
351
	set_fs(oldfs);
352
	return 0;
353 354
}

355 356 357 358 359 360 361 362 363 364
static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
{
	kfree(vq->indirect);
	vq->indirect = NULL;
	kfree(vq->log);
	vq->log = NULL;
	kfree(vq->heads);
	vq->heads = NULL;
}

Jason Wang's avatar
Jason Wang committed
365 366 367
/* Helper to allocate iovec buffers for all vqs. */
static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
{
368
	struct vhost_virtqueue *vq;
Jason Wang's avatar
Jason Wang committed
369
	int i;
370

Jason Wang's avatar
Jason Wang committed
371
	for (i = 0; i < dev->nvqs; ++i) {
372 373 374 375 376 377
		vq = dev->vqs[i];
		vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV,
				       GFP_KERNEL);
		vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL);
		vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL);
		if (!vq->indirect || !vq->log || !vq->heads)
Jason Wang's avatar
Jason Wang committed
378 379 380
			goto err_nomem;
	}
	return 0;
381

Jason Wang's avatar
Jason Wang committed
382
err_nomem:
383
	for (; i >= 0; --i)
384
		vhost_vq_free_iovecs(dev->vqs[i]);
Jason Wang's avatar
Jason Wang committed
385 386 387 388 389 390
	return -ENOMEM;
}

static void vhost_dev_free_iovecs(struct vhost_dev *dev)
{
	int i;
391

392
	for (i = 0; i < dev->nvqs; ++i)
393
		vhost_vq_free_iovecs(dev->vqs[i]);
Jason Wang's avatar
Jason Wang committed
394 395
}

Zhi Yong Wu's avatar
Zhi Yong Wu committed
396
void vhost_dev_init(struct vhost_dev *dev,
397
		    struct vhost_virtqueue **vqs, int nvqs)
398
{
399
	struct vhost_virtqueue *vq;
400
	int i;
401

402 403 404 405 406
	dev->vqs = vqs;
	dev->nvqs = nvqs;
	mutex_init(&dev->mutex);
	dev->log_ctx = NULL;
	dev->log_file = NULL;
407
	dev->umem = NULL;
Jason Wang's avatar
Jason Wang committed
408
	dev->iotlb = NULL;
409
	dev->mm = NULL;
410
	dev->worker = NULL;
Jason Wang's avatar
Jason Wang committed
411
	init_llist_head(&dev->work_list);
Jason Wang's avatar
Jason Wang committed
412 413 414 415
	init_waitqueue_head(&dev->wait);
	INIT_LIST_HEAD(&dev->read_list);
	INIT_LIST_HEAD(&dev->pending_list);
	spin_lock_init(&dev->iotlb_lock);
Jason Wang's avatar
Jason Wang committed
416

417 418

	for (i = 0; i < dev->nvqs; ++i) {
419 420 421 422 423 424 425 426 427 428
		vq = dev->vqs[i];
		vq->log = NULL;
		vq->indirect = NULL;
		vq->heads = NULL;
		vq->dev = dev;
		mutex_init(&vq->mutex);
		vhost_vq_reset(dev, vq);
		if (vq->handle_kick)
			vhost_poll_init(&vq->poll, vq->handle_kick,
					POLLIN, dev);
429 430
	}
}
431
EXPORT_SYMBOL_GPL(vhost_dev_init);
432 433 434 435 436 437 438

/* Caller should have device mutex */
long vhost_dev_check_owner(struct vhost_dev *dev)
{
	/* Are you the owner? If not, I don't think you mean to do that */
	return dev->mm == current->mm ? 0 : -EPERM;
}
439
EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
440

441
struct vhost_attach_cgroups_struct {
442 443 444
	struct vhost_work work;
	struct task_struct *owner;
	int ret;
445 446 447 448
};

static void vhost_attach_cgroups_work(struct vhost_work *work)
{
449 450 451 452
	struct vhost_attach_cgroups_struct *s;

	s = container_of(work, struct vhost_attach_cgroups_struct, work);
	s->ret = cgroup_attach_task_all(s->owner, current);
453 454 455 456
}

static int vhost_attach_cgroups(struct vhost_dev *dev)
{
457 458 459 460 461 462 463
	struct vhost_attach_cgroups_struct attach;

	attach.owner = current;
	vhost_work_init(&attach.work, vhost_attach_cgroups_work);
	vhost_work_queue(dev, &attach.work);
	vhost_work_flush(dev, &attach.work);
	return attach.ret;
464 465
}

466 467 468 469 470
/* Caller should have device mutex */
bool vhost_dev_has_owner(struct vhost_dev *dev)
{
	return dev->mm;
}
471
EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
472

473
/* Caller should have device mutex */
Asias He's avatar
Asias He committed
474
long vhost_dev_set_owner(struct vhost_dev *dev)
475
{
476 477
	struct task_struct *worker;
	int err;
478

479
	/* Is there an owner already? */
480
	if (vhost_dev_has_owner(dev)) {
481 482 483
		err = -EBUSY;
		goto err_mm;
	}
484

485 486
	/* No owner, become one */
	dev->mm = get_task_mm(current);
487 488 489 490 491 492 493
	worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
	if (IS_ERR(worker)) {
		err = PTR_ERR(worker);
		goto err_worker;
	}

	dev->worker = worker;
494 495 496
	wake_up_process(worker);	/* avoid contributing to loadavg */

	err = vhost_attach_cgroups(dev);
497 498
	if (err)
		goto err_cgroup;
499

Jason Wang's avatar
Jason Wang committed
500 501 502 503
	err = vhost_dev_alloc_iovecs(dev);
	if (err)
		goto err_cgroup;

504
	return 0;
505 506
err_cgroup:
	kthread_stop(worker);
507
	dev->worker = NULL;
508 509 510 511 512 513
err_worker:
	if (dev->mm)
		mmput(dev->mm);
	dev->mm = NULL;
err_mm:
	return err;
514
}
515
EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
516

517
static void *vhost_kvzalloc(unsigned long size)
518
{
519 520 521 522 523 524 525 526 527 528
	void *n = kzalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);

	if (!n)
		n = vzalloc(size);
	return n;
}

struct vhost_umem *vhost_dev_reset_owner_prepare(void)
{
	return vhost_kvzalloc(sizeof(struct vhost_umem));
529
}
530
EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
531

532
/* Caller should have device mutex */
533
void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem)
534
{
535 536
	int i;

537
	vhost_dev_cleanup(dev, true);
538

539
	/* Restore memory to default empty mapping. */
540 541
	INIT_LIST_HEAD(&umem->umem_list);
	dev->umem = umem;
542 543 544 545
	/* We don't need VQ locks below since vhost_dev_cleanup makes sure
	 * VQs aren't running.
	 */
	for (i = 0; i < dev->nvqs; ++i)
546
		dev->vqs[i]->umem = umem;
547
}
548
EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
549

550
void vhost_dev_stop(struct vhost_dev *dev)
551 552
{
	int i;
553 554

	for (i = 0; i < dev->nvqs; ++i) {
555 556 557
		if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) {
			vhost_poll_stop(&dev->vqs[i]->poll);
			vhost_poll_flush(&dev->vqs[i]->poll);
558
		}
559 560
	}
}
561
EXPORT_SYMBOL_GPL(vhost_dev_stop);
562

Jason Wang's avatar
Jason Wang committed
563 564 565 566 567 568 569 570 571
static void vhost_umem_free(struct vhost_umem *umem,
			    struct vhost_umem_node *node)
{
	vhost_umem_interval_tree_remove(node, &umem->umem_tree);
	list_del(&node->link);
	kfree(node);
	umem->numem--;
}

572 573 574 575 576 577 578
static void vhost_umem_clean(struct vhost_umem *umem)
{
	struct vhost_umem_node *node, *tmp;

	if (!umem)
		return;

Jason Wang's avatar
Jason Wang committed
579 580 581
	list_for_each_entry_safe(node, tmp, &umem->umem_list, link)
		vhost_umem_free(umem, node);

582 583 584
	kvfree(umem);
}

Jason Wang's avatar
Jason Wang committed
585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
static void vhost_clear_msg(struct vhost_dev *dev)
{
	struct vhost_msg_node *node, *n;

	spin_lock(&dev->iotlb_lock);

	list_for_each_entry_safe(node, n, &dev->read_list, node) {
		list_del(&node->node);
		kfree(node);
	}

	list_for_each_entry_safe(node, n, &dev->pending_list, node) {
		list_del(&node->node);
		kfree(node);
	}

	spin_unlock(&dev->iotlb_lock);
}

604 605
/* Caller should have device mutex if and only if locked is set */
void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
606 607
{
	int i;
608

609
	for (i = 0; i < dev->nvqs; ++i) {
610 611 612 613 614 615 616 617 618 619 620
		if (dev->vqs[i]->error_ctx)
			eventfd_ctx_put(dev->vqs[i]->error_ctx);
		if (dev->vqs[i]->error)
			fput(dev->vqs[i]->error);
		if (dev->vqs[i]->kick)
			fput(dev->vqs[i]->kick);
		if (dev->vqs[i]->call_ctx)
			eventfd_ctx_put(dev->vqs[i]->call_ctx);
		if (dev->vqs[i]->call)
			fput(dev->vqs[i]->call);
		vhost_vq_reset(dev, dev->vqs[i]);
621
	}
Jason Wang's avatar
Jason Wang committed
622
	vhost_dev_free_iovecs(dev);
623 624 625 626 627 628 629
	if (dev->log_ctx)
		eventfd_ctx_put(dev->log_ctx);
	dev->log_ctx = NULL;
	if (dev->log_file)
		fput(dev->log_file);
	dev->log_file = NULL;
	/* No one will access memory at this point */
630 631
	vhost_umem_clean(dev->umem);
	dev->umem = NULL;
Jason Wang's avatar
Jason Wang committed
632 633 634 635
	vhost_umem_clean(dev->iotlb);
	dev->iotlb = NULL;
	vhost_clear_msg(dev);
	wake_up_interruptible_poll(&dev->wait, POLLIN | POLLRDNORM);
Jason Wang's avatar
Jason Wang committed
636
	WARN_ON(!llist_empty(&dev->work_list));
637 638 639 640
	if (dev->worker) {
		kthread_stop(dev->worker);
		dev->worker = NULL;
	}
641 642 643
	if (dev->mm)
		mmput(dev->mm);
	dev->mm = NULL;
644
}
645
EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
646 647 648 649

static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
{
	u64 a = addr / VHOST_PAGE_SIZE / 8;
650

651 652 653
	/* Make sure 64 bit math will not overflow. */
	if (a > ULONG_MAX - (unsigned long)log_base ||
	    a + (unsigned long)log_base > ULONG_MAX)
654
		return 0;
655 656 657 658 659

	return access_ok(VERIFY_WRITE, log_base + a,
			 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
}

660 661 662 663 664 665
static bool vhost_overflow(u64 uaddr, u64 size)
{
	/* Make sure 64 bit math will not overflow. */
	return uaddr > ULONG_MAX || size > ULONG_MAX || uaddr > ULONG_MAX - size;
}

666
/* Caller should have vq mutex and device mutex. */
667
static int vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem,
668 669
			       int log_all)
{
670
	struct vhost_umem_node *node;
671

672
	if (!umem)
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
673
		return 0;
674

675 676 677
	list_for_each_entry(node, &umem->umem_list, link) {
		unsigned long a = node->userspace_addr;

678
		if (vhost_overflow(node->userspace_addr, node->size))
679
			return 0;
680 681 682


		if (!access_ok(VERIFY_WRITE, (void __user *)a,
683
				    node->size))
684 685
			return 0;
		else if (log_all && !log_access_ok(log_base,
686 687
						   node->start,
						   node->size))
688 689 690 691 692 693 694
			return 0;
	}
	return 1;
}

/* Can we switch to this memory table? */
/* Caller should have device mutex but not vq mutex */
695
static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
696 697 698
			    int log_all)
{
	int i;
699

700 701
	for (i = 0; i < d->nvqs; ++i) {
		int ok;
702 703
		bool log;

704
		mutex_lock(&d->vqs[i]->mutex);
705
		log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL);
706
		/* If ring is inactive, will check when it's enabled. */
707
		if (d->vqs[i]->private_data)
708 709
			ok = vq_memory_access_ok(d->vqs[i]->log_base,
						 umem, log);
710 711
		else
			ok = 1;
712
		mutex_unlock(&d->vqs[i]->mutex);
713 714 715 716 717 718
		if (!ok)
			return 0;
	}
	return 1;
}

Jason Wang's avatar
Jason Wang committed
719 720
static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
			  struct iovec iov[], int iov_size, int access);
721 722 723 724

static int vhost_copy_to_user(struct vhost_virtqueue *vq, void *to,
			      const void *from, unsigned size)
{
Jason Wang's avatar
Jason Wang committed
725
	int ret;
726

Jason Wang's avatar
Jason Wang committed
727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749
	if (!vq->iotlb)
		return __copy_to_user(to, from, size);
	else {
		/* This function should be called after iotlb
		 * prefetch, which means we're sure that all vq
		 * could be access through iotlb. So -EAGAIN should
		 * not happen in this case.
		 */
		/* TODO: more fast path */
		struct iov_iter t;
		ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
				     ARRAY_SIZE(vq->iotlb_iov),
				     VHOST_ACCESS_WO);
		if (ret < 0)
			goto out;
		iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size);
		ret = copy_to_iter(from, size, &t);
		if (ret == size)
			ret = 0;
	}
out:
	return ret;
}
750 751 752 753

static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,
				void *from, unsigned size)
{
Jason Wang's avatar
Jason Wang committed
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
	int ret;

	if (!vq->iotlb)
		return __copy_from_user(to, from, size);
	else {
		/* This function should be called after iotlb
		 * prefetch, which means we're sure that vq
		 * could be access through iotlb. So -EAGAIN should
		 * not happen in this case.
		 */
		/* TODO: more fast path */
		struct iov_iter f;
		ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
				     ARRAY_SIZE(vq->iotlb_iov),
				     VHOST_ACCESS_RO);
		if (ret < 0) {
			vq_err(vq, "IOTLB translation failure: uaddr "
			       "%p size 0x%llx\n", from,
			       (unsigned long long) size);
			goto out;
		}
		iov_iter_init(&f, READ, vq->iotlb_iov, ret, size);
		ret = copy_from_iter(to, size, &f);
		if (ret == size)
			ret = 0;
	}

out:
	return ret;
}

static void __user *__vhost_get_user(struct vhost_virtqueue *vq,
				     void *addr, unsigned size)
{
	int ret;

	/* This function should be called after iotlb
	 * prefetch, which means we're sure that vq
	 * could be access through iotlb. So -EAGAIN should
	 * not happen in this case.
	 */
	/* TODO: more fast path */
	ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
			     ARRAY_SIZE(vq->iotlb_iov),
			     VHOST_ACCESS_RO);
	if (ret < 0) {
		vq_err(vq, "IOTLB translation failure: uaddr "
			"%p size 0x%llx\n", addr,
			(unsigned long long) size);
		return NULL;
	}

	if (ret != 1 || vq->iotlb_iov[0].iov_len != size) {
		vq_err(vq, "Non atomic userspace memory access: uaddr "
			"%p size 0x%llx\n", addr,
			(unsigned long long) size);
		return NULL;
	}

	return vq->iotlb_iov[0].iov_base;
}

#define vhost_put_user(vq, x, ptr) \
({ \
	int ret = -EFAULT; \
	if (!vq->iotlb) { \
		ret = __put_user(x, ptr); \
	} else { \
		__typeof__(ptr) to = \
			(__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \
		if (to != NULL) \
			ret = __put_user(x, to); \
		else \
			ret = -EFAULT;	\
	} \
	ret; \
})

#define vhost_get_user(vq, x, ptr) \
({ \
	int ret; \
	if (!vq->iotlb) { \
		ret = __get_user(x, ptr); \
	} else { \
		__typeof__(ptr) from = \
			(__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \
		if (from != NULL) \
			ret = __get_user(x, from); \
		else \
			ret = -EFAULT; \
	} \
	ret; \
})

static void vhost_dev_lock_vqs(struct vhost_dev *d)
{
	int i = 0;
	for (i = 0; i < d->nvqs; ++i)
		mutex_lock(&d->vqs[i]->mutex);
}

static void vhost_dev_unlock_vqs(struct vhost_dev *d)
{
	int i = 0;
	for (i = 0; i < d->nvqs; ++i)
		mutex_unlock(&d->vqs[i]->mutex);
}

static int vhost_new_umem_range(struct vhost_umem *umem,
				u64 start, u64 size, u64 end,
				u64 userspace_addr, int perm)
{
	struct vhost_umem_node *tmp, *node = kmalloc(sizeof(*node), GFP_ATOMIC);

	if (!node)
		return -ENOMEM;

	if (umem->numem == max_iotlb_entries) {
		tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link);
		vhost_umem_free(umem, tmp);
	}

	node->start = start;
	node->size = size;
	node->last = end;
	node->userspace_addr = userspace_addr;
	node->perm = perm;
	INIT_LIST_HEAD(&node->link);
	list_add_tail(&node->link, &umem->umem_list);
	vhost_umem_interval_tree_insert(node, &umem->umem_tree);
	umem->numem++;

	return 0;
}

static void vhost_del_umem_range(struct vhost_umem *umem,
				 u64 start, u64 end)
{
	struct vhost_umem_node *node;

	while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
							   start, end)))
		vhost_umem_free(umem, node);
}

static void vhost_iotlb_notify_vq(struct vhost_dev *d,
				  struct vhost_iotlb_msg *msg)
{
	struct vhost_msg_node *node, *n;

	spin_lock(&d->iotlb_lock);

	list_for_each_entry_safe(node, n, &d->pending_list, node) {
		struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb;
		if (msg->iova <= vq_msg->iova &&
		    msg->iova + msg->size - 1 > vq_msg->iova &&
		    vq_msg->type == VHOST_IOTLB_MISS) {
			vhost_poll_queue(&node->vq->poll);
			list_del(&node->node);
			kfree(node);
		}
	}

	spin_unlock(&d->iotlb_lock);
}

static int umem_access_ok(u64 uaddr, u64 size, int access)
{
	unsigned long a = uaddr;

924 925 926 927
	/* Make sure 64 bit math will not overflow. */
	if (vhost_overflow(uaddr, size))
		return -EFAULT;

Jason Wang's avatar
Jason Wang committed
928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
	if ((access & VHOST_ACCESS_RO) &&
	    !access_ok(VERIFY_READ, (void __user *)a, size))
		return -EFAULT;
	if ((access & VHOST_ACCESS_WO) &&
	    !access_ok(VERIFY_WRITE, (void __user *)a, size))
		return -EFAULT;
	return 0;
}

int vhost_process_iotlb_msg(struct vhost_dev *dev,
			    struct vhost_iotlb_msg *msg)
{
	int ret = 0;

	vhost_dev_lock_vqs(dev);
	switch (msg->type) {
	case VHOST_IOTLB_UPDATE:
		if (!dev->iotlb) {
			ret = -EFAULT;
			break;
		}
		if (umem_access_ok(msg->uaddr, msg->size, msg->perm)) {
			ret = -EFAULT;
			break;
		}
		if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
					 msg->iova + msg->size - 1,
					 msg->uaddr, msg->perm)) {
			ret = -ENOMEM;
			break;
		}
		vhost_iotlb_notify_vq(dev, msg);
		break;
	case VHOST_IOTLB_INVALIDATE:
		vhost_del_umem_range(dev->iotlb, msg->iova,
				     msg->iova + msg->size - 1);
		break;
	default:
		ret = -EINVAL;
		break;
	}

	vhost_dev_unlock_vqs(dev);
	return ret;
}
ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
			     struct iov_iter *from)
{
	struct vhost_msg_node node;
	unsigned size = sizeof(struct vhost_msg);
	size_t ret;
	int err;

	if (iov_iter_count(from) < size)
		return 0;
	ret = copy_from_iter(&node.msg, size, from);
	if (ret != size)
		goto done;

	switch (node.msg.type) {
	case VHOST_IOTLB_MSG:
		err = vhost_process_iotlb_msg(dev, &node.msg.iotlb);
		if (err)
			ret = err;
		break;
	default:
		ret = -EINVAL;
		break;
	}

done:
	return ret;
}
EXPORT_SYMBOL(vhost_chr_write_iter);

unsigned int vhost_chr_poll(struct file *file, struct vhost_dev *dev,
			    poll_table *wait)
{
	unsigned int mask = 0;

	poll_wait(file, &dev->wait, wait);

	if (!list_empty(&dev->read_list))
		mask |= POLLIN | POLLRDNORM;

	return mask;
}
EXPORT_SYMBOL(vhost_chr_poll);

ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
			    int noblock)
{
	DEFINE_WAIT(wait);
	struct vhost_msg_node *node;
	ssize_t ret = 0;
	unsigned size = sizeof(struct vhost_msg);

	if (iov_iter_count(to) < size)
		return 0;

	while (1) {
		if (!noblock)
			prepare_to_wait(&dev->wait, &wait,
					TASK_INTERRUPTIBLE);

		node = vhost_dequeue_msg(dev, &dev->read_list);
		if (node)
			break;
		if (noblock) {
			ret = -EAGAIN;
			break;
		}
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
		if (!dev->iotlb) {
			ret = -EBADFD;
			break;
		}

		schedule();
	}

	if (!noblock)
		finish_wait(&dev->wait, &wait);

	if (node) {
		ret = copy_to_iter(&node->msg, size, to);

		if (ret != size || node->msg.type != VHOST_IOTLB_MISS) {
			kfree(node);
			return ret;
		}

		vhost_enqueue_msg(dev, &dev->pending_list, node);
	}

	return ret;
}
EXPORT_SYMBOL_GPL(vhost_chr_read_iter);

static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
{
	struct vhost_dev *dev = vq->dev;
	struct vhost_msg_node *node;
	struct vhost_iotlb_msg *msg;

	node = vhost_new_msg(vq, VHOST_IOTLB_MISS);
	if (!node)
		return -ENOMEM;

	msg = &node->msg.iotlb;
	msg->type = VHOST_IOTLB_MISS;
	msg->iova = iova;
	msg->perm = access;

	vhost_enqueue_msg(dev, &dev->read_list, node);

	return 0;
1088 1089
}

1090
static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1091 1092 1093
			struct vring_desc __user *desc,
			struct vring_avail __user *avail,
			struct vring_used __user *used)
Jason Wang's avatar
Jason Wang committed
1094

1095
{
1096
	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
Jason Wang's avatar
Jason Wang committed
1097

1098 1099
	return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
	       access_ok(VERIFY_READ, avail,
1100
			 sizeof *avail + num * sizeof *avail->ring + s) &&
1101
	       access_ok(VERIFY_WRITE, used,
1102
			sizeof *used + num * sizeof *used->ring + s);
1103 1104
}

Jason Wang's avatar
Jason Wang committed
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
static int iotlb_access_ok(struct vhost_virtqueue *vq,
			   int access, u64 addr, u64 len)
{
	const struct vhost_umem_node *node;
	struct vhost_umem *umem = vq->iotlb;
	u64 s = 0, size;

	while (len > s) {
		node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
							   addr,
							   addr + len - 1);
		if (node == NULL || node->start > addr) {
			vhost_iotlb_miss(vq, addr, access);
			return false;
		} else if (!(node->perm & access)) {
			/* Report the possible access violation by
			 * request another translation from userspace.
			 */
			return false;
		}

		size = node->size - addr + node->start;
		s += size;
		addr += size;
	}

	return true;
}

int vq_iotlb_prefetch(struct vhost_virtqueue *vq)
{
	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
	unsigned int num = vq->num;

	if (!vq->iotlb)
		return 1;

	return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
			       num * sizeof *vq->desc) &&
	       iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
			       sizeof *vq->avail +
			       num * sizeof *vq->avail->ring + s) &&
	       iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
			       sizeof *vq->used +
			       num * sizeof *vq->used->ring + s);
}
EXPORT_SYMBOL_GPL(vq_iotlb_prefetch);

1153 1154 1155 1156
/* Can we log writes? */
/* Caller should have device mutex but not vq mutex */
int vhost_log_access_ok(struct vhost_dev *dev)
{
1157
	return memory_access_ok(dev, dev->umem, 1);
1158
}
1159
EXPORT_SYMBOL_GPL(vhost_log_access_ok);
1160 1161 1162

/* Verify access for write logging. */
/* Caller should have vq mutex and device mutex */
1163
static int vq_log_access_ok(struct vhost_virtqueue *vq,
1164
			    void __user *log_base)
1165
{
1166
	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
Arnd Bergmann's avatar
Arnd Bergmann committed
1167

1168
	return vq_memory_access_ok(log_base, vq->umem,
1169
				   vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
1170 1171
		(!vq->log_used || log_access_ok(log_base, vq->log_addr,
					sizeof *vq->used +
1172
					vq->num * sizeof *vq->used->ring + s));
1173 1174 1175 1176 1177 1178
}

/* Can we start vq? */
/* Caller should have vq mutex and device mutex */
int vhost_vq_access_ok(struct vhost_virtqueue *vq)
{
Jason Wang's avatar
Jason Wang committed
1179 1180 1181 1182 1183 1184
	if (vq->iotlb) {
		/* When device IOTLB was used, the access validation
		 * will be validated during prefetching.
		 */
		return 1;
	}
1185 1186
	return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used) &&
		vq_log_access_ok(vq, vq->log_base);
1187
}
1188
EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
1189

Jason Wang's avatar
Jason Wang committed
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
static struct vhost_umem *vhost_umem_alloc(void)
{
	struct vhost_umem *umem = vhost_kvzalloc(sizeof(*umem));

	if (!umem)
		return NULL;

	umem->umem_tree = RB_ROOT;
	umem->numem = 0;
	INIT_LIST_HEAD(&umem->umem_list);

	return umem;
}

1204 1205
static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
{
1206 1207 1208
	struct vhost_memory mem, *newmem;
	struct vhost_memory_region *region;
	struct vhost_umem *newumem, *oldumem;
1209
	unsigned long size = offsetof(struct vhost_memory, regions);
1210
	int i;
1211

1212 1213
	if (copy_from_user(&mem, m, size))
		return -EFAULT;
1214 1215
	if (mem.padding)
		return -EOPNOTSUPP;
1216
	if (mem.nregions > max_mem_regions)
1217
		return -E2BIG;
1218
	newmem = vhost_kvzalloc(size + mem.nregions * sizeof(*m->regions));
1219 1220 1221 1222
	if (!newmem)
		return -ENOMEM;

	memcpy(newmem, &mem, size);
1223 1224
	if (copy_from_user(newmem->regions, m->regions,
			   mem.nregions * sizeof *m->regions)) {
1225
		kvfree(newmem);
1226
		return -EFAULT;
1227 1228
	}

Jason Wang's avatar
Jason Wang committed
1229
	newumem = vhost_umem_alloc();
1230
	if (!newumem) {
1231
		kvfree(newmem);
1232 1233 1234 1235 1236 1237
		return -ENOMEM;
	}

	for (region = newmem->regions;
	     region < newmem->regions + mem.nregions;
	     region++) {
Jason Wang's avatar
Jason Wang committed
1238 1239 1240 1241 1242 1243 1244
		if (vhost_new_umem_range(newumem,
					 region->guest_phys_addr,
					 region->memory_size,
					 region->guest_phys_addr +
					 region->memory_size - 1,
					 region->userspace_addr,
					 VHOST_ACCESS_RW))
1245
			goto err;
1246
	}
1247 1248 1249 1250 1251 1252

	if (!memory_access_ok(d, newumem, 0))
		goto err;

	oldumem = d->umem;
	d->umem = newumem;
1253

1254
	/* All memory accesses are done under some VQ mutex. */
1255 1256
	for (i = 0; i < d->nvqs; ++i) {
		mutex_lock(&d->vqs[i]->mutex);
1257
		d->vqs[i]->umem = newumem;
1258 1259
		mutex_unlock(&d->vqs[i]->mutex);
	}
1260 1261 1262

	kvfree(newmem);
	vhost_umem_clean(oldumem);
1263
	return 0;
1264 1265 1266 1267 1268

err:
	vhost_umem_clean(newumem);
	kvfree(newmem);
	return -EFAULT;
1269 1270
}

1271
long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp)
1272
{
1273 1274
	struct file *eventfp, *filep = NULL;
	bool pollstart = false, pollstop = false;
1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
	struct eventfd_ctx *ctx = NULL;
	u32 __user *idxp = argp;
	struct vhost_virtqueue *vq;
	struct vhost_vring_state s;
	struct vhost_vring_file f;
	struct vhost_vring_addr a;
	u32 idx;
	long r;

	r = get_user(idx, idxp);
	if (r < 0)
		return r;
1287
	if (idx >= d->nvqs)
1288 1289
		return -ENOBUFS;

1290
	vq = d->vqs[idx];
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301

	mutex_lock(&vq->mutex);

	switch (ioctl) {
	case VHOST_SET_VRING_NUM:
		/* Resizing ring with an active backend?
		 * You don't want to do that. */
		if (vq->private_data) {
			r = -EBUSY;
			break;
		}
1302 1303
		if (copy_from_user(&s, argp, sizeof s)) {
			r = -EFAULT;
1304
			break;
1305
		}
1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
		if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
			r = -EINVAL;
			break;
		}
		vq->num = s.num;
		break;
	case VHOST_SET_VRING_BASE:
		/* Moving base with an active backend?
		 * You don't want to do that. */
		if (vq->private_data) {
			r = -EBUSY;
			break;
		}
1319 1320
		if (copy_from_user(&s, argp, sizeof s)) {
			r = -EFAULT;
1321
			break;
1322
		}
1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
		if (s.num > 0xffff) {
			r = -EINVAL;
			break;
		}
		vq->last_avail_idx = s.num;
		/* Forget the cached index value. */
		vq->avail_idx = vq->last_avail_idx;
		break;
	case VHOST_GET_VRING_BASE:
		s.index = idx;
		s.num = vq->last_avail_idx;
1334 1335
		if (copy_to_user(argp, &s, sizeof s))
			r = -EFAULT;
1336 1337
		break;
	case VHOST_SET_VRING_ADDR:
1338 1339
		if (copy_from_user(&a, argp, sizeof a)) {
			r = -EFAULT;
1340
			break;
1341
		}
1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
		if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
			r = -EOPNOTSUPP;
			break;
		}
		/* For 32bit, verify that the top 32bits of the user
		   data are set to zero. */
		if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
		    (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
		    (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
			r = -EFAULT;
			break;
		}
1354 1355 1356 1357 1358 1359

		/* Make sure it's safe to cast pointers to vring types. */
		BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
		BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
		if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
		    (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
1360
		    (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) {
1361 1362 1363 1364 1365 1366 1367 1368
			r = -EINVAL;
			break;
		}

		/* We only verify access here if backend is configured.
		 * If it is not, we don't as size might not have been setup.
		 * We will verify when backend is configured. */
		if (vq->private_data) {
1369
			if (!vq_access_ok(vq, vq->num,
1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
				(void __user *)(unsigned long)a.desc_user_addr,
				(void __user *)(unsigned long)a.avail_user_addr,
				(void __user *)(unsigned long)a.used_user_addr)) {
				r = -EINVAL;
				break;
			}

			/* Also validate log access for used ring if enabled. */
			if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
			    !log_access_ok(vq->log_base, a.log_guest_addr,
					   sizeof *vq->used +
					   vq->num * sizeof *vq->used->ring)) {
				r = -EINVAL;
				break;
			}
		}

		vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
		vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
		vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
		vq->log_addr = a.log_guest_addr;
		vq->used = (void __user *)(unsigned long)a.used_user_addr;
		break;
	case VHOST_SET_VRING_KICK:
1394 1395
		if (copy_from_user(&f, argp, sizeof f)) {
			r = -EFAULT;
1396
			break;
1397
		}