vhost.c 56.3 KB
Newer Older
1 2 3 4 5 6
/* Copyright (C) 2009 Red Hat, Inc.
 * Copyright (C) 2006 Rusty Russell IBM Corporation
 *
 * Author: Michael S. Tsirkin <mst@redhat.com>
 *
 * Inspiration, some code, and most witty comments come from
Rob Landley's avatar
Rob Landley committed
7
 * Documentation/virtual/lguest/lguest.c, by Rusty Russell
8 9 10 11 12 13 14 15
 *
 * This work is licensed under the terms of the GNU GPL, version 2.
 *
 * Generic code for virtio server in host kernel.
 */

#include <linux/eventfd.h>
#include <linux/vhost.h>
16
#include <linux/uio.h>
17
#include <linux/mm.h>
18
#include <linux/mmu_context.h>
19 20 21 22 23
#include <linux/miscdevice.h>
#include <linux/mutex.h>
#include <linux/poll.h>
#include <linux/file.h>
#include <linux/highmem.h>
24
#include <linux/slab.h>
25
#include <linux/vmalloc.h>
26
#include <linux/kthread.h>
27
#include <linux/cgroup.h>
28
#include <linux/module.h>
29
#include <linux/sort.h>
30
#include <linux/interval_tree_generic.h>
31 32 33

#include "vhost.h"

34 35 36 37
static ushort max_mem_regions = 64;
module_param(max_mem_regions, ushort, 0444);
MODULE_PARM_DESC(max_mem_regions,
	"Maximum number of memory regions in memory map. (default: 64)");
Jason Wang's avatar
Jason Wang committed
38 39 40 41
static int max_iotlb_entries = 2048;
module_param(max_iotlb_entries, int, 0444);
MODULE_PARM_DESC(max_iotlb_entries,
	"Maximum number of iotlb entries. (default: 2048)");
42

43 44 45 46
enum {
	VHOST_MEMORY_F_LOG = 0x1,
};

47 48
#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
49

50 51 52 53
INTERVAL_TREE_DEFINE(struct vhost_umem_node,
		     rb, __u64, __subtree_last,
		     START, LAST, , vhost_umem_interval_tree);

54
#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
55
static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
56 57 58 59
{
	vq->user_be = !virtio_legacy_is_little_endian();
}

60 61 62 63 64 65 66 67 68 69
static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq)
{
	vq->user_be = true;
}

static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq)
{
	vq->user_be = false;
}

70 71 72 73 74 75 76 77 78 79 80 81 82 83
static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
{
	struct vhost_vring_state s;

	if (vq->private_data)
		return -EBUSY;

	if (copy_from_user(&s, argp, sizeof(s)))
		return -EFAULT;

	if (s.num != VHOST_VRING_LITTLE_ENDIAN &&
	    s.num != VHOST_VRING_BIG_ENDIAN)
		return -EINVAL;

84 85 86 87
	if (s.num == VHOST_VRING_BIG_ENDIAN)
		vhost_enable_cross_endian_big(vq);
	else
		vhost_enable_cross_endian_little(vq);
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115

	return 0;
}

static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
				   int __user *argp)
{
	struct vhost_vring_state s = {
		.index = idx,
		.num = vq->user_be
	};

	if (copy_to_user(argp, &s, sizeof(s)))
		return -EFAULT;

	return 0;
}

static void vhost_init_is_le(struct vhost_virtqueue *vq)
{
	/* Note for legacy virtio: user_be is initialized at reset time
	 * according to the host endianness. If userspace does not set an
	 * explicit endianness, the default behavior is native endian, as
	 * expected by legacy virtio.
	 */
	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;
}
#else
116
static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
{
}

static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
{
	return -ENOIOCTLCMD;
}

static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
				   int __user *argp)
{
	return -ENOIOCTLCMD;
}

static void vhost_init_is_le(struct vhost_virtqueue *vq)
{
	if (vhost_has_feature(vq, VIRTIO_F_VERSION_1))
		vq->is_le = true;
}
#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */

138 139 140 141 142
static void vhost_reset_is_le(struct vhost_virtqueue *vq)
{
	vq->is_le = virtio_legacy_is_little_endian();
}

Jason Wang's avatar
Jason Wang committed
143 144 145 146 147 148 149 150 151 152 153 154 155
struct vhost_flush_struct {
	struct vhost_work work;
	struct completion wait_event;
};

static void vhost_flush_work(struct vhost_work *work)
{
	struct vhost_flush_struct *s;

	s = container_of(work, struct vhost_flush_struct, work);
	complete(&s->wait_event);
}

156 157 158 159 160
static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
			    poll_table *pt)
{
	struct vhost_poll *poll;

161
	poll = container_of(pt, struct vhost_poll, table);
162 163 164 165 166 167 168
	poll->wqh = wqh;
	add_wait_queue(wqh, &poll->wait);
}

static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync,
			     void *key)
{
169 170
	struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);

171 172 173
	if (!((unsigned long)key & poll->mask))
		return 0;

174
	vhost_poll_queue(poll);
175 176 177
	return 0;
}

178
void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
179
{
Jason Wang's avatar
Jason Wang committed
180
	clear_bit(VHOST_WORK_QUEUED, &work->flags);
181 182 183
	work->fn = fn;
	init_waitqueue_head(&work->done);
}
184
EXPORT_SYMBOL_GPL(vhost_work_init);
185

186
/* Init poll structure */
187 188
void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
		     unsigned long mask, struct vhost_dev *dev)
189 190 191 192
{
	init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
	init_poll_funcptr(&poll->table, vhost_poll_func);
	poll->mask = mask;
193
	poll->dev = dev;
194
	poll->wqh = NULL;
195

196
	vhost_work_init(&poll->work, fn);
197
}
198
EXPORT_SYMBOL_GPL(vhost_poll_init);
199 200 201

/* Start polling a file. We add ourselves to file's wait queue. The caller must
 * keep a reference to a file until after vhost_poll_stop is called. */
202
int vhost_poll_start(struct vhost_poll *poll, struct file *file)
203 204
{
	unsigned long mask;
205
	int ret = 0;
206

207 208 209
	if (poll->wqh)
		return 0;

210 211 212
	mask = file->f_op->poll(file, &poll->table);
	if (mask)
		vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask);
213 214 215 216 217 218 219
	if (mask & POLLERR) {
		if (poll->wqh)
			remove_wait_queue(poll->wqh, &poll->wait);
		ret = -EINVAL;
	}

	return ret;
220
}
221
EXPORT_SYMBOL_GPL(vhost_poll_start);
222 223 224 225 226

/* Stop polling a file. After this function returns, it becomes safe to drop the
 * file reference. You must also flush afterwards. */
void vhost_poll_stop(struct vhost_poll *poll)
{
227 228 229 230
	if (poll->wqh) {
		remove_wait_queue(poll->wqh, &poll->wait);
		poll->wqh = NULL;
	}
231
}
232
EXPORT_SYMBOL_GPL(vhost_poll_stop);
233

Jason Wang's avatar
Jason Wang committed
234
void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
235
{
Jason Wang's avatar
Jason Wang committed
236
	struct vhost_flush_struct flush;
237

Jason Wang's avatar
Jason Wang committed
238 239 240
	if (dev->worker) {
		init_completion(&flush.wait_event);
		vhost_work_init(&flush.work, vhost_flush_work);
241

Jason Wang's avatar
Jason Wang committed
242 243 244
		vhost_work_queue(dev, &flush.work);
		wait_for_completion(&flush.wait_event);
	}
245
}
246
EXPORT_SYMBOL_GPL(vhost_work_flush);
247

248 249 250 251 252 253
/* Flush any work that has been scheduled. When calling this, don't hold any
 * locks that are also used by the callback. */
void vhost_poll_flush(struct vhost_poll *poll)
{
	vhost_work_flush(poll->dev, &poll->work);
}
254
EXPORT_SYMBOL_GPL(vhost_poll_flush);
255

256
void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
257
{
Jason Wang's avatar
Jason Wang committed
258 259
	if (!dev->worker)
		return;
260

Jason Wang's avatar
Jason Wang committed
261 262 263 264 265 266
	if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) {
		/* We can only add the work to the list after we're
		 * sure it was not in the list.
		 */
		smp_mb();
		llist_add(&work->node, &dev->work_list);
267 268
		wake_up_process(dev->worker);
	}
269
}
270
EXPORT_SYMBOL_GPL(vhost_work_queue);
271

272 273 274
/* A lockless hint for busy polling code to exit the loop */
bool vhost_has_work(struct vhost_dev *dev)
{
Jason Wang's avatar
Jason Wang committed
275
	return !llist_empty(&dev->work_list);
276 277 278
}
EXPORT_SYMBOL_GPL(vhost_has_work);

279 280 281 282
void vhost_poll_queue(struct vhost_poll *poll)
{
	vhost_work_queue(poll->dev, &poll->work);
}
283
EXPORT_SYMBOL_GPL(vhost_poll_queue);
284

285 286 287 288 289 290 291 292 293 294
static void vhost_vq_reset(struct vhost_dev *dev,
			   struct vhost_virtqueue *vq)
{
	vq->num = 1;
	vq->desc = NULL;
	vq->avail = NULL;
	vq->used = NULL;
	vq->last_avail_idx = 0;
	vq->avail_idx = 0;
	vq->last_used_idx = 0;
295 296
	vq->signalled_used = 0;
	vq->signalled_used_valid = false;
297 298 299 300
	vq->used_flags = 0;
	vq->log_used = false;
	vq->log_addr = -1ull;
	vq->private_data = NULL;
301
	vq->acked_features = 0;
302 303 304 305 306 307
	vq->log_base = NULL;
	vq->error_ctx = NULL;
	vq->error = NULL;
	vq->kick = NULL;
	vq->call_ctx = NULL;
	vq->call = NULL;
308
	vq->log_ctx = NULL;
309 310
	vhost_reset_is_le(vq);
	vhost_disable_cross_endian(vq);
311
	vq->busyloop_timeout = 0;
312
	vq->umem = NULL;
Jason Wang's avatar
Jason Wang committed
313
	vq->iotlb = NULL;
314 315
}

316 317 318
static int vhost_worker(void *data)
{
	struct vhost_dev *dev = data;
Jason Wang's avatar
Jason Wang committed
319 320
	struct vhost_work *work, *work_next;
	struct llist_node *node;
321
	mm_segment_t oldfs = get_fs();
322

323
	set_fs(USER_DS);
324 325
	use_mm(dev->mm);

326 327 328 329 330 331
	for (;;) {
		/* mb paired w/ kthread_stop */
		set_current_state(TASK_INTERRUPTIBLE);

		if (kthread_should_stop()) {
			__set_current_state(TASK_RUNNING);
332
			break;
333
		}
Jason Wang's avatar
Jason Wang committed
334 335 336 337 338 339 340 341 342 343

		node = llist_del_all(&dev->work_list);
		if (!node)
			schedule();

		node = llist_reverse_order(node);
		/* make sure flag is seen after deletion */
		smp_wmb();
		llist_for_each_entry_safe(work, work_next, node, node) {
			clear_bit(VHOST_WORK_QUEUED, &work->flags);
344 345
			__set_current_state(TASK_RUNNING);
			work->fn(work);
346 347
			if (need_resched())
				schedule();
Jason Wang's avatar
Jason Wang committed
348
		}
349
	}
350
	unuse_mm(dev->mm);
351
	set_fs(oldfs);
352
	return 0;
353 354
}

355 356 357 358 359 360 361 362 363 364
static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
{
	kfree(vq->indirect);
	vq->indirect = NULL;
	kfree(vq->log);
	vq->log = NULL;
	kfree(vq->heads);
	vq->heads = NULL;
}

Jason Wang's avatar
Jason Wang committed
365 366 367
/* Helper to allocate iovec buffers for all vqs. */
static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
{
368
	struct vhost_virtqueue *vq;
Jason Wang's avatar
Jason Wang committed
369
	int i;
370

Jason Wang's avatar
Jason Wang committed
371
	for (i = 0; i < dev->nvqs; ++i) {
372 373 374 375 376 377
		vq = dev->vqs[i];
		vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV,
				       GFP_KERNEL);
		vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL);
		vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL);
		if (!vq->indirect || !vq->log || !vq->heads)
Jason Wang's avatar
Jason Wang committed
378 379 380
			goto err_nomem;
	}
	return 0;
381

Jason Wang's avatar
Jason Wang committed
382
err_nomem:
383
	for (; i >= 0; --i)
384
		vhost_vq_free_iovecs(dev->vqs[i]);
Jason Wang's avatar
Jason Wang committed
385 386 387 388 389 390
	return -ENOMEM;
}

static void vhost_dev_free_iovecs(struct vhost_dev *dev)
{
	int i;
391

392
	for (i = 0; i < dev->nvqs; ++i)
393
		vhost_vq_free_iovecs(dev->vqs[i]);
Jason Wang's avatar
Jason Wang committed
394 395
}

Zhi Yong Wu's avatar
Zhi Yong Wu committed
396
void vhost_dev_init(struct vhost_dev *dev,
397
		    struct vhost_virtqueue **vqs, int nvqs)
398
{
399
	struct vhost_virtqueue *vq;
400
	int i;
401

402 403 404 405 406
	dev->vqs = vqs;
	dev->nvqs = nvqs;
	mutex_init(&dev->mutex);
	dev->log_ctx = NULL;
	dev->log_file = NULL;
407
	dev->umem = NULL;
Jason Wang's avatar
Jason Wang committed
408
	dev->iotlb = NULL;
409
	dev->mm = NULL;
410
	dev->worker = NULL;
Jason Wang's avatar
Jason Wang committed
411
	init_llist_head(&dev->work_list);
Jason Wang's avatar
Jason Wang committed
412 413 414 415
	init_waitqueue_head(&dev->wait);
	INIT_LIST_HEAD(&dev->read_list);
	INIT_LIST_HEAD(&dev->pending_list);
	spin_lock_init(&dev->iotlb_lock);
Jason Wang's avatar
Jason Wang committed
416

417 418

	for (i = 0; i < dev->nvqs; ++i) {
419 420 421 422 423 424 425 426 427 428
		vq = dev->vqs[i];
		vq->log = NULL;
		vq->indirect = NULL;
		vq->heads = NULL;
		vq->dev = dev;
		mutex_init(&vq->mutex);
		vhost_vq_reset(dev, vq);
		if (vq->handle_kick)
			vhost_poll_init(&vq->poll, vq->handle_kick,
					POLLIN, dev);
429 430
	}
}
431
EXPORT_SYMBOL_GPL(vhost_dev_init);
432 433 434 435 436 437 438

/* Caller should have device mutex */
long vhost_dev_check_owner(struct vhost_dev *dev)
{
	/* Are you the owner? If not, I don't think you mean to do that */
	return dev->mm == current->mm ? 0 : -EPERM;
}
439
EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
440

441
struct vhost_attach_cgroups_struct {
442 443 444
	struct vhost_work work;
	struct task_struct *owner;
	int ret;
445 446 447 448
};

static void vhost_attach_cgroups_work(struct vhost_work *work)
{
449 450 451 452
	struct vhost_attach_cgroups_struct *s;

	s = container_of(work, struct vhost_attach_cgroups_struct, work);
	s->ret = cgroup_attach_task_all(s->owner, current);
453 454 455 456
}

static int vhost_attach_cgroups(struct vhost_dev *dev)
{
457 458 459 460 461 462 463
	struct vhost_attach_cgroups_struct attach;

	attach.owner = current;
	vhost_work_init(&attach.work, vhost_attach_cgroups_work);
	vhost_work_queue(dev, &attach.work);
	vhost_work_flush(dev, &attach.work);
	return attach.ret;
464 465
}

466 467 468 469 470
/* Caller should have device mutex */
bool vhost_dev_has_owner(struct vhost_dev *dev)
{
	return dev->mm;
}
471
EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
472

473
/* Caller should have device mutex */
Asias He's avatar
Asias He committed
474
long vhost_dev_set_owner(struct vhost_dev *dev)
475
{
476 477
	struct task_struct *worker;
	int err;
478

479
	/* Is there an owner already? */
480
	if (vhost_dev_has_owner(dev)) {
481 482 483
		err = -EBUSY;
		goto err_mm;
	}
484

485 486
	/* No owner, become one */
	dev->mm = get_task_mm(current);
487 488 489 490 491 492 493
	worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
	if (IS_ERR(worker)) {
		err = PTR_ERR(worker);
		goto err_worker;
	}

	dev->worker = worker;
494 495 496
	wake_up_process(worker);	/* avoid contributing to loadavg */

	err = vhost_attach_cgroups(dev);
497 498
	if (err)
		goto err_cgroup;
499

Jason Wang's avatar
Jason Wang committed
500 501 502 503
	err = vhost_dev_alloc_iovecs(dev);
	if (err)
		goto err_cgroup;

504
	return 0;
505 506
err_cgroup:
	kthread_stop(worker);
507
	dev->worker = NULL;
508 509 510 511 512 513
err_worker:
	if (dev->mm)
		mmput(dev->mm);
	dev->mm = NULL;
err_mm:
	return err;
514
}
515
EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
516

517
static void *vhost_kvzalloc(unsigned long size)
518
{
519 520 521 522 523 524 525 526 527 528
	void *n = kzalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);

	if (!n)
		n = vzalloc(size);
	return n;
}

struct vhost_umem *vhost_dev_reset_owner_prepare(void)
{
	return vhost_kvzalloc(sizeof(struct vhost_umem));
529
}
530
EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
531

532
/* Caller should have device mutex */
533
void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem)
534
{
535 536
	int i;

537
	vhost_dev_cleanup(dev, true);
538

539
	/* Restore memory to default empty mapping. */
540 541
	INIT_LIST_HEAD(&umem->umem_list);
	dev->umem = umem;
542 543 544 545
	/* We don't need VQ locks below since vhost_dev_cleanup makes sure
	 * VQs aren't running.
	 */
	for (i = 0; i < dev->nvqs; ++i)
546
		dev->vqs[i]->umem = umem;
547
}
548
EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
549

550
void vhost_dev_stop(struct vhost_dev *dev)
551 552
{
	int i;
553 554

	for (i = 0; i < dev->nvqs; ++i) {
555 556 557
		if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) {
			vhost_poll_stop(&dev->vqs[i]->poll);
			vhost_poll_flush(&dev->vqs[i]->poll);
558
		}
559 560
	}
}
561
EXPORT_SYMBOL_GPL(vhost_dev_stop);
562

Jason Wang's avatar
Jason Wang committed
563 564 565 566 567 568 569 570 571
static void vhost_umem_free(struct vhost_umem *umem,
			    struct vhost_umem_node *node)
{
	vhost_umem_interval_tree_remove(node, &umem->umem_tree);
	list_del(&node->link);
	kfree(node);
	umem->numem--;
}

572 573 574 575 576 577 578
static void vhost_umem_clean(struct vhost_umem *umem)
{
	struct vhost_umem_node *node, *tmp;

	if (!umem)
		return;

Jason Wang's avatar
Jason Wang committed
579 580 581
	list_for_each_entry_safe(node, tmp, &umem->umem_list, link)
		vhost_umem_free(umem, node);

582 583 584
	kvfree(umem);
}

Jason Wang's avatar
Jason Wang committed
585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
static void vhost_clear_msg(struct vhost_dev *dev)
{
	struct vhost_msg_node *node, *n;

	spin_lock(&dev->iotlb_lock);

	list_for_each_entry_safe(node, n, &dev->read_list, node) {
		list_del(&node->node);
		kfree(node);
	}

	list_for_each_entry_safe(node, n, &dev->pending_list, node) {
		list_del(&node->node);
		kfree(node);
	}

	spin_unlock(&dev->iotlb_lock);
}

604 605
/* Caller should have device mutex if and only if locked is set */
void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
606 607
{
	int i;
608

609
	for (i = 0; i < dev->nvqs; ++i) {
610 611 612 613 614 615 616 617 618 619 620
		if (dev->vqs[i]->error_ctx)
			eventfd_ctx_put(dev->vqs[i]->error_ctx);
		if (dev->vqs[i]->error)
			fput(dev->vqs[i]->error);
		if (dev->vqs[i]->kick)
			fput(dev->vqs[i]->kick);
		if (dev->vqs[i]->call_ctx)
			eventfd_ctx_put(dev->vqs[i]->call_ctx);
		if (dev->vqs[i]->call)
			fput(dev->vqs[i]->call);
		vhost_vq_reset(dev, dev->vqs[i]);
621
	}
Jason Wang's avatar
Jason Wang committed
622
	vhost_dev_free_iovecs(dev);
623 624 625 626 627 628 629
	if (dev->log_ctx)
		eventfd_ctx_put(dev->log_ctx);
	dev->log_ctx = NULL;
	if (dev->log_file)
		fput(dev->log_file);
	dev->log_file = NULL;
	/* No one will access memory at this point */
630 631
	vhost_umem_clean(dev->umem);
	dev->umem = NULL;
Jason Wang's avatar
Jason Wang committed
632 633 634 635
	vhost_umem_clean(dev->iotlb);
	dev->iotlb = NULL;
	vhost_clear_msg(dev);
	wake_up_interruptible_poll(&dev->wait, POLLIN | POLLRDNORM);
Jason Wang's avatar
Jason Wang committed
636
	WARN_ON(!llist_empty(&dev->work_list));
637 638 639 640
	if (dev->worker) {
		kthread_stop(dev->worker);
		dev->worker = NULL;
	}
641 642 643
	if (dev->mm)
		mmput(dev->mm);
	dev->mm = NULL;
644
}
645
EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
646 647 648 649

static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
{
	u64 a = addr / VHOST_PAGE_SIZE / 8;
650

651 652 653
	/* Make sure 64 bit math will not overflow. */
	if (a > ULONG_MAX - (unsigned long)log_base ||
	    a + (unsigned long)log_base > ULONG_MAX)
654
		return 0;
655 656 657 658 659 660

	return access_ok(VERIFY_WRITE, log_base + a,
			 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
}

/* Caller should have vq mutex and device mutex. */
661
static int vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem,
662 663
			       int log_all)
{
664
	struct vhost_umem_node *node;
665

666
	if (!umem)
Michael S. Tsirkin's avatar
Michael S. Tsirkin committed
667
		return 0;
668

669 670 671 672
	list_for_each_entry(node, &umem->umem_list, link) {
		unsigned long a = node->userspace_addr;

		if (node->size > ULONG_MAX)
673 674
			return 0;
		else if (!access_ok(VERIFY_WRITE, (void __user *)a,
675
				    node->size))
676 677
			return 0;
		else if (log_all && !log_access_ok(log_base,
678 679
						   node->start,
						   node->size))
680 681 682 683 684 685 686
			return 0;
	}
	return 1;
}

/* Can we switch to this memory table? */
/* Caller should have device mutex but not vq mutex */
687
static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
688 689 690
			    int log_all)
{
	int i;
691

692 693
	for (i = 0; i < d->nvqs; ++i) {
		int ok;
694 695
		bool log;

696
		mutex_lock(&d->vqs[i]->mutex);
697
		log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL);
698
		/* If ring is inactive, will check when it's enabled. */
699
		if (d->vqs[i]->private_data)
700 701
			ok = vq_memory_access_ok(d->vqs[i]->log_base,
						 umem, log);
702 703
		else
			ok = 1;
704
		mutex_unlock(&d->vqs[i]->mutex);
705 706 707 708 709 710
		if (!ok)
			return 0;
	}
	return 1;
}

Jason Wang's avatar
Jason Wang committed
711 712
static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
			  struct iovec iov[], int iov_size, int access);
713 714 715 716

static int vhost_copy_to_user(struct vhost_virtqueue *vq, void *to,
			      const void *from, unsigned size)
{
Jason Wang's avatar
Jason Wang committed
717
	int ret;
718

Jason Wang's avatar
Jason Wang committed
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741
	if (!vq->iotlb)
		return __copy_to_user(to, from, size);
	else {
		/* This function should be called after iotlb
		 * prefetch, which means we're sure that all vq
		 * could be access through iotlb. So -EAGAIN should
		 * not happen in this case.
		 */
		/* TODO: more fast path */
		struct iov_iter t;
		ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
				     ARRAY_SIZE(vq->iotlb_iov),
				     VHOST_ACCESS_WO);
		if (ret < 0)
			goto out;
		iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size);
		ret = copy_to_iter(from, size, &t);
		if (ret == size)
			ret = 0;
	}
out:
	return ret;
}
742 743 744 745

static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,
				void *from, unsigned size)
{
Jason Wang's avatar
Jason Wang committed
746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
	int ret;

	if (!vq->iotlb)
		return __copy_from_user(to, from, size);
	else {
		/* This function should be called after iotlb
		 * prefetch, which means we're sure that vq
		 * could be access through iotlb. So -EAGAIN should
		 * not happen in this case.
		 */
		/* TODO: more fast path */
		struct iov_iter f;
		ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
				     ARRAY_SIZE(vq->iotlb_iov),
				     VHOST_ACCESS_RO);
		if (ret < 0) {
			vq_err(vq, "IOTLB translation failure: uaddr "
			       "%p size 0x%llx\n", from,
			       (unsigned long long) size);
			goto out;
		}
		iov_iter_init(&f, READ, vq->iotlb_iov, ret, size);
		ret = copy_from_iter(to, size, &f);
		if (ret == size)
			ret = 0;
	}

out:
	return ret;
}

static void __user *__vhost_get_user(struct vhost_virtqueue *vq,
				     void *addr, unsigned size)
{
	int ret;

	/* This function should be called after iotlb
	 * prefetch, which means we're sure that vq
	 * could be access through iotlb. So -EAGAIN should
	 * not happen in this case.
	 */
	/* TODO: more fast path */
	ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
			     ARRAY_SIZE(vq->iotlb_iov),
			     VHOST_ACCESS_RO);
	if (ret < 0) {
		vq_err(vq, "IOTLB translation failure: uaddr "
			"%p size 0x%llx\n", addr,
			(unsigned long long) size);
		return NULL;
	}

	if (ret != 1 || vq->iotlb_iov[0].iov_len != size) {
		vq_err(vq, "Non atomic userspace memory access: uaddr "
			"%p size 0x%llx\n", addr,
			(unsigned long long) size);
		return NULL;
	}

	return vq->iotlb_iov[0].iov_base;
}

#define vhost_put_user(vq, x, ptr) \
({ \
	int ret = -EFAULT; \
	if (!vq->iotlb) { \
		ret = __put_user(x, ptr); \
	} else { \
		__typeof__(ptr) to = \
			(__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \
		if (to != NULL) \
			ret = __put_user(x, to); \
		else \
			ret = -EFAULT;	\
	} \
	ret; \
})

#define vhost_get_user(vq, x, ptr) \
({ \
	int ret; \
	if (!vq->iotlb) { \
		ret = __get_user(x, ptr); \
	} else { \
		__typeof__(ptr) from = \
			(__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \
		if (from != NULL) \
			ret = __get_user(x, from); \
		else \
			ret = -EFAULT; \
	} \
	ret; \
})

static void vhost_dev_lock_vqs(struct vhost_dev *d)
{
	int i = 0;
	for (i = 0; i < d->nvqs; ++i)
		mutex_lock(&d->vqs[i]->mutex);
}

static void vhost_dev_unlock_vqs(struct vhost_dev *d)
{
	int i = 0;
	for (i = 0; i < d->nvqs; ++i)
		mutex_unlock(&d->vqs[i]->mutex);
}

static int vhost_new_umem_range(struct vhost_umem *umem,
				u64 start, u64 size, u64 end,
				u64 userspace_addr, int perm)
{
	struct vhost_umem_node *tmp, *node = kmalloc(sizeof(*node), GFP_ATOMIC);

	if (!node)
		return -ENOMEM;

	if (umem->numem == max_iotlb_entries) {
		tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link);
		vhost_umem_free(umem, tmp);
	}

	node->start = start;
	node->size = size;
	node->last = end;
	node->userspace_addr = userspace_addr;
	node->perm = perm;
	INIT_LIST_HEAD(&node->link);
	list_add_tail(&node->link, &umem->umem_list);
	vhost_umem_interval_tree_insert(node, &umem->umem_tree);
	umem->numem++;

	return 0;
}

static void vhost_del_umem_range(struct vhost_umem *umem,
				 u64 start, u64 end)
{
	struct vhost_umem_node *node;

	while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
							   start, end)))
		vhost_umem_free(umem, node);
}

static void vhost_iotlb_notify_vq(struct vhost_dev *d,
				  struct vhost_iotlb_msg *msg)
{
	struct vhost_msg_node *node, *n;

	spin_lock(&d->iotlb_lock);

	list_for_each_entry_safe(node, n, &d->pending_list, node) {
		struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb;
		if (msg->iova <= vq_msg->iova &&
		    msg->iova + msg->size - 1 > vq_msg->iova &&
		    vq_msg->type == VHOST_IOTLB_MISS) {
			vhost_poll_queue(&node->vq->poll);
			list_del(&node->node);
			kfree(node);
		}
	}

	spin_unlock(&d->iotlb_lock);
}

static int umem_access_ok(u64 uaddr, u64 size, int access)
{
	unsigned long a = uaddr;

	if ((access & VHOST_ACCESS_RO) &&
	    !access_ok(VERIFY_READ, (void __user *)a, size))
		return -EFAULT;
	if ((access & VHOST_ACCESS_WO) &&
	    !access_ok(VERIFY_WRITE, (void __user *)a, size))
		return -EFAULT;
	return 0;
}

int vhost_process_iotlb_msg(struct vhost_dev *dev,
			    struct vhost_iotlb_msg *msg)
{
	int ret = 0;

	vhost_dev_lock_vqs(dev);
	switch (msg->type) {
	case VHOST_IOTLB_UPDATE:
		if (!dev->iotlb) {
			ret = -EFAULT;
			break;
		}
		if (umem_access_ok(msg->uaddr, msg->size, msg->perm)) {
			ret = -EFAULT;
			break;
		}
		if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
					 msg->iova + msg->size - 1,
					 msg->uaddr, msg->perm)) {
			ret = -ENOMEM;
			break;
		}
		vhost_iotlb_notify_vq(dev, msg);
		break;
	case VHOST_IOTLB_INVALIDATE:
		vhost_del_umem_range(dev->iotlb, msg->iova,
				     msg->iova + msg->size - 1);
		break;
	default:
		ret = -EINVAL;
		break;
	}

	vhost_dev_unlock_vqs(dev);
	return ret;
}
ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
			     struct iov_iter *from)
{
	struct vhost_msg_node node;
	unsigned size = sizeof(struct vhost_msg);
	size_t ret;
	int err;

	if (iov_iter_count(from) < size)
		return 0;
	ret = copy_from_iter(&node.msg, size, from);
	if (ret != size)
		goto done;

	switch (node.msg.type) {
	case VHOST_IOTLB_MSG:
		err = vhost_process_iotlb_msg(dev, &node.msg.iotlb);
		if (err)
			ret = err;
		break;
	default:
		ret = -EINVAL;
		break;
	}

done:
	return ret;
}
EXPORT_SYMBOL(vhost_chr_write_iter);

unsigned int vhost_chr_poll(struct file *file, struct vhost_dev *dev,
			    poll_table *wait)
{
	unsigned int mask = 0;

	poll_wait(file, &dev->wait, wait);

	if (!list_empty(&dev->read_list))
		mask |= POLLIN | POLLRDNORM;

	return mask;
}
EXPORT_SYMBOL(vhost_chr_poll);

ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
			    int noblock)
{
	DEFINE_WAIT(wait);
	struct vhost_msg_node *node;
	ssize_t ret = 0;
	unsigned size = sizeof(struct vhost_msg);

	if (iov_iter_count(to) < size)
		return 0;

	while (1) {
		if (!noblock)
			prepare_to_wait(&dev->wait, &wait,
					TASK_INTERRUPTIBLE);

		node = vhost_dequeue_msg(dev, &dev->read_list);
		if (node)
			break;
		if (noblock) {
			ret = -EAGAIN;
			break;
		}
		if (signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}
		if (!dev->iotlb) {
			ret = -EBADFD;
			break;
		}

		schedule();
	}

	if (!noblock)
		finish_wait(&dev->wait, &wait);

	if (node) {
		ret = copy_to_iter(&node->msg, size, to);

		if (ret != size || node->msg.type != VHOST_IOTLB_MISS) {
			kfree(node);
			return ret;
		}

		vhost_enqueue_msg(dev, &dev->pending_list, node);
	}

	return ret;
}
EXPORT_SYMBOL_GPL(vhost_chr_read_iter);

static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
{
	struct vhost_dev *dev = vq->dev;
	struct vhost_msg_node *node;
	struct vhost_iotlb_msg *msg;

	node = vhost_new_msg(vq, VHOST_IOTLB_MISS);
	if (!node)
		return -ENOMEM;

	msg = &node->msg.iotlb;
	msg->type = VHOST_IOTLB_MISS;
	msg->iova = iova;
	msg->perm = access;

	vhost_enqueue_msg(dev, &dev->read_list, node);

	return 0;
1076 1077
}

1078
static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1079 1080 1081
			struct vring_desc __user *desc,
			struct vring_avail __user *avail,
			struct vring_used __user *used)
Jason Wang's avatar
Jason Wang committed
1082

1083
{
1084
	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
Jason Wang's avatar
Jason Wang committed
1085

1086 1087
	return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
	       access_ok(VERIFY_READ, avail,
1088
			 sizeof *avail + num * sizeof *avail->ring + s) &&
1089
	       access_ok(VERIFY_WRITE, used,
1090
			sizeof *used + num * sizeof *used->ring + s);
1091 1092
}

Jason Wang's avatar
Jason Wang committed
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140
static int iotlb_access_ok(struct vhost_virtqueue *vq,
			   int access, u64 addr, u64 len)
{
	const struct vhost_umem_node *node;
	struct vhost_umem *umem = vq->iotlb;
	u64 s = 0, size;

	while (len > s) {
		node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
							   addr,
							   addr + len - 1);
		if (node == NULL || node->start > addr) {
			vhost_iotlb_miss(vq, addr, access);
			return false;
		} else if (!(node->perm & access)) {
			/* Report the possible access violation by
			 * request another translation from userspace.
			 */
			return false;
		}

		size = node->size - addr + node->start;
		s += size;
		addr += size;
	}

	return true;
}

int vq_iotlb_prefetch(struct vhost_virtqueue *vq)
{
	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
	unsigned int num = vq->num;

	if (!vq->iotlb)
		return 1;

	return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
			       num * sizeof *vq->desc) &&
	       iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
			       sizeof *vq->avail +
			       num * sizeof *vq->avail->ring + s) &&
	       iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
			       sizeof *vq->used +
			       num * sizeof *vq->used->ring + s);
}
EXPORT_SYMBOL_GPL(vq_iotlb_prefetch);

1141 1142 1143 1144
/* Can we log writes? */
/* Caller should have device mutex but not vq mutex */
int vhost_log_access_ok(struct vhost_dev *dev)
{
1145
	return memory_access_ok(dev, dev->umem, 1);
1146
}
1147
EXPORT_SYMBOL_GPL(vhost_log_access_ok);
1148 1149 1150

/* Verify access for write logging. */
/* Caller should have vq mutex and device mutex */
1151
static int vq_log_access_ok(struct vhost_virtqueue *vq,
1152
			    void __user *log_base)
1153
{
1154
	size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
Arnd Bergmann's avatar
Arnd Bergmann committed
1155

1156
	return vq_memory_access_ok(log_base, vq->umem,
1157
				   vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
1158 1159
		(!vq->log_used || log_access_ok(log_base, vq->log_addr,
					sizeof *vq->used +
1160
					vq->num * sizeof *vq->used->ring + s));
1161 1162 1163 1164 1165 1166
}

/* Can we start vq? */
/* Caller should have vq mutex and device mutex */
int vhost_vq_access_ok(struct vhost_virtqueue *vq)
{
Jason Wang's avatar
Jason Wang committed
1167 1168 1169 1170 1171 1172
	if (vq->iotlb) {
		/* When device IOTLB was used, the access validation
		 * will be validated during prefetching.
		 */
		return 1;
	}
1173 1174
	return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used) &&
		vq_log_access_ok(vq, vq->log_base);
1175
}
1176
EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
1177

Jason Wang's avatar
Jason Wang committed
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
static struct vhost_umem *vhost_umem_alloc(void)
{
	struct vhost_umem *umem = vhost_kvzalloc(sizeof(*umem));

	if (!umem)
		return NULL;

	umem->umem_tree = RB_ROOT;
	umem->numem = 0;
	INIT_LIST_HEAD(&umem->umem_list);

	return umem;
}

1192 1193
static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
{
1194 1195 1196
	struct vhost_memory mem, *newmem;
	struct vhost_memory_region *region;
	struct vhost_umem *newumem, *oldumem;
1197
	unsigned long size = offsetof(struct vhost_memory, regions);
1198
	int i;
1199

1200 1201
	if (copy_from_user(&mem, m, size))
		return -EFAULT;
1202 1203
	if (mem.padding)
		return -EOPNOTSUPP;
1204
	if (mem.nregions > max_mem_regions)
1205
		return -E2BIG;
1206
	newmem = vhost_kvzalloc(size + mem.nregions * sizeof(*m->regions));
1207 1208 1209 1210
	if (!newmem)
		return -ENOMEM;

	memcpy(newmem, &mem, size);
1211 1212
	if (copy_from_user(newmem->regions, m->regions,
			   mem.nregions * sizeof *m->regions)) {
1213
		kvfree(newmem);
1214
		return -EFAULT;
1215 1216
	}

Jason Wang's avatar
Jason Wang committed
1217
	newumem = vhost_umem_alloc();
1218
	if (!newumem) {
1219
		kvfree(newmem);
1220 1221 1222 1223 1224 1225
		return -ENOMEM;
	}

	for (region = newmem->regions;
	     region < newmem->regions + mem.nregions;
	     region++) {
Jason Wang's avatar
Jason Wang committed
1226 1227 1228 1229 1230 1231 1232
		if (vhost_new_umem_range(newumem,
					 region->guest_phys_addr,
					 region->memory_size,
					 region->guest_phys_addr +
					 region->memory_size - 1,
					 region->userspace_addr,
					 VHOST_ACCESS_RW))
1233
			goto err;
1234
	}
1235 1236 1237 1238 1239 1240

	if (!memory_access_ok(d, newumem, 0))
		goto err;

	oldumem = d->umem;
	d->umem = newumem;
1241

1242
	/* All memory accesses are done under some VQ mutex. */
1243 1244
	for (i = 0; i < d->nvqs; ++i) {
		mutex_lock(&d->vqs[i]->mutex);
1245
		d->vqs[i]->umem = newumem;
1246 1247
		mutex_unlock(&d->vqs[i]->mutex);
	}
1248 1249 1250

	kvfree(newmem);
	vhost_umem_clean(oldumem);
1251
	return 0;
1252 1253 1254 1255 1256

err:
	vhost_umem_clean(newumem);
	kvfree(newmem);
	return -EFAULT;
1257 1258
}

1259
long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp)
1260
{
1261 1262
	struct file *eventfp, *filep = NULL;
	bool pollstart = false, pollstop = false;
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
	struct eventfd_ctx *ctx = NULL;
	u32 __user *idxp = argp;
	struct vhost_virtqueue *vq;
	struct vhost_vring_state s;
	struct vhost_vring_file f;
	struct vhost_vring_addr a;
	u32 idx;
	long r;

	r = get_user(idx, idxp);
	if (r < 0)
		return r;
1275
	if (idx >= d->nvqs)
1276 1277
		return -ENOBUFS;

1278
	vq = d->vqs[idx];
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289

	mutex_lock(&vq->mutex);

	switch (ioctl) {
	case VHOST_SET_VRING_NUM:
		/* Resizing ring with an active backend?
		 * You don't want to do that. */
		if (vq->private_data) {
			r = -EBUSY;
			break;
		}
1290 1291
		if (copy_from_user(&s, argp, sizeof s)) {
			r = -EFAULT;
1292
			break;
1293
		}
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
		if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
			r = -EINVAL;
			break;
		}
		vq->num = s.num;
		break;
	case VHOST_SET_VRING_BASE:
		/* Moving base with an active backend?
		 * You don't want to do that. */
		if (vq->private_data) {
			r = -EBUSY;
			break;
		}
1307 1308
		if (copy_from_user(&s, argp, sizeof s)) {
			r = -EFAULT;
1309
			break;
1310
		}
1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
		if (s.num > 0xffff) {
			r = -EINVAL;
			break;
		}
		vq->last_avail_idx = s.num;
		/* Forget the cached index value. */
		vq->avail_idx = vq->last_avail_idx;
		break;
	case VHOST_GET_VRING_BASE:
		s.index = idx;
		s.num = vq->last_avail_idx;
1322 1323
		if (copy_to_user(argp, &s, sizeof s))
			r = -EFAULT;
1324 1325
		break;
	case VHOST_SET_VRING_ADDR:
1326 1327
		if (copy_from_user(&a, argp, sizeof a)) {
			r = -EFAULT;
1328
			break;
1329
		}
1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341
		if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
			r = -EOPNOTSUPP;
			break;
		}
		/* For 32bit, verify that the top 32bits of the user
		   data are set to zero. */
		if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
		    (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
		    (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
			r = -EFAULT;
			break;
		}
1342 1343 1344 1345 1346 1347

		/* Make sure it's safe to cast pointers to vring types. */
		BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
		BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
		if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
		    (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
1348
		    (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) {
1349 1350 1351 1352 1353 1354 1355 1356
			r = -EINVAL;
			break;
		}

		/* We only verify access here if backend is configured.
		 * If it is not, we don't as size might not have been setup.
		 * We will verify when backend is configured. */
		if (vq->private_data) {
1357
			if (!vq_access_ok(vq, vq->num,
1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381
				(void __user *)(unsigned long)a.desc_user_addr,
				(void __user *)(unsigned long)a.avail_user_addr,
				(void __user *)(unsigned long)a.used_user_addr)) {
				r = -EINVAL;
				break;
			}

			/* Also validate log access for used ring if enabled. */
			if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
			    !log_access_ok(vq->log_base, a.log_guest_addr,
					   sizeof *vq->used +
					   vq->num * sizeof *vq->used->ring)) {
				r = -EINVAL;
				break;
			}
		}

		vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
		vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
		vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
		vq->log_addr = a.log_guest_addr;
		vq->used = (void __user *)(unsigned long)a.used_user_addr;
		break;
	case VHOST_SET_VRING_KICK:
1382 1383
		if (copy_from_user(&f, argp, sizeof f)) {
			r = -EFAULT;
1384
			break;
1385
		}
1386
		eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
1387 1388 1389 1390
		if (IS_ERR(eventfp)) {
			r = PTR_ERR(eventfp);
			break;
		}
1391
		if (eventfp != vq->kick) {
1392 1393
			pollstop = (filep = vq->kick) != NULL;
			pollstart = (vq->kick = eventfp) != NULL;
1394 1395 1396 1397
		} else
			filep = eventfp;
		break;
	case VHOST_SET_VRING_CALL: