i915_gem.c 106 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

#include "drmP.h"
#include "drm.h"
#include "i915_drm.h"
#include "i915_drv.h"
Chris Wilson's avatar
Chris Wilson committed
32
#include "i915_trace.h"
33
#include "intel_drv.h"
34
#include <linux/shmem_fs.h>
35
#include <linux/slab.h>
36
#include <linux/swap.h>
37
#include <linux/pci.h>
38
#include <linux/dma-buf.h>
39

40
static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
41
42
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
43
44
45
static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
						    unsigned alignment,
						    bool map_and_fenceable);
46
47
static int i915_gem_phys_pwrite(struct drm_device *dev,
				struct drm_i915_gem_object *obj,
48
				struct drm_i915_gem_pwrite *args,
49
				struct drm_file *file);
50

51
52
53
54
55
56
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj);
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable);

57
static int i915_gem_inactive_shrink(struct shrinker *shrinker,
58
				    struct shrink_control *sc);
59
static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
60

61
62
63
64
65
66
67
68
static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
{
	if (obj->tiling_mode)
		i915_gem_release_mmap(obj);

	/* As we do not have an associated fence register, we will force
	 * a tiling change if we ever need to acquire one.
	 */
69
	obj->fence_dirty = false;
70
71
72
	obj->fence_reg = I915_FENCE_REG_NONE;
}

73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
}

88
89
static int
i915_gem_wait_for_error(struct drm_device *dev)
90
91
92
93
94
95
96
97
98
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct completion *x = &dev_priv->error_completion;
	unsigned long flags;
	int ret;

	if (!atomic_read(&dev_priv->mm.wedged))
		return 0;

99
100
101
102
103
104
105
106
107
108
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
109
		return ret;
110
	}
111

112
113
114
115
116
117
118
119
120
121
122
	if (atomic_read(&dev_priv->mm.wedged)) {
		/* GPU is hung, bump the completion count to account for
		 * the token we just consumed so that we never hit zero and
		 * end up waiting upon a subsequent completion event that
		 * will never happen.
		 */
		spin_lock_irqsave(&x->wait.lock, flags);
		x->done++;
		spin_unlock_irqrestore(&x->wait.lock, flags);
	}
	return 0;
123
124
}

125
int i915_mutex_lock_interruptible(struct drm_device *dev)
126
127
128
{
	int ret;

129
	ret = i915_gem_wait_for_error(dev);
130
131
132
133
134
135
136
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

137
	WARN_ON(i915_verify_lists(dev));
138
139
	return 0;
}
140

141
static inline bool
142
i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
143
{
144
	return !obj->active;
145
146
}

147
148
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
149
		    struct drm_file *file)
150
151
{
	struct drm_i915_gem_init *args = data;
152

153
154
155
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return -ENODEV;

156
157
158
	if (args->gtt_start >= args->gtt_end ||
	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
		return -EINVAL;
159

160
161
162
163
	/* GEM with user mode setting was never supported on ilk and later. */
	if (INTEL_INFO(dev)->gen >= 5)
		return -ENODEV;

164
	mutex_lock(&dev->struct_mutex);
165
166
	i915_gem_init_global_gtt(dev, args->gtt_start,
				 args->gtt_end, args->gtt_end);
167
168
	mutex_unlock(&dev->struct_mutex);

169
	return 0;
170
171
}

172
173
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
174
			    struct drm_file *file)
175
{
176
	struct drm_i915_private *dev_priv = dev->dev_private;
177
	struct drm_i915_gem_get_aperture *args = data;
178
179
	struct drm_i915_gem_object *obj;
	size_t pinned;
180

181
	pinned = 0;
182
	mutex_lock(&dev->struct_mutex);
183
184
185
	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
		if (obj->pin_count)
			pinned += obj->gtt_space->size;
186
	mutex_unlock(&dev->struct_mutex);
187

188
	args->aper_size = dev_priv->mm.gtt_total;
189
	args->aper_available_size = args->aper_size - pinned;
190

191
192
193
	return 0;
}

194
195
196
197
198
static int
i915_gem_create(struct drm_file *file,
		struct drm_device *dev,
		uint64_t size,
		uint32_t *handle_p)
199
{
200
	struct drm_i915_gem_object *obj;
201
202
	int ret;
	u32 handle;
203

204
	size = roundup(size, PAGE_SIZE);
205
206
	if (size == 0)
		return -EINVAL;
207
208

	/* Allocate the new object */
209
	obj = i915_gem_alloc_object(dev, size);
210
211
212
	if (obj == NULL)
		return -ENOMEM;

213
	ret = drm_gem_handle_create(file, &obj->base, &handle);
214
	if (ret) {
215
216
		drm_gem_object_release(&obj->base);
		i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
217
		kfree(obj);
218
		return ret;
219
	}
220

221
	/* drop reference from allocate - handle holds it now */
222
	drm_gem_object_unreference(&obj->base);
223
224
	trace_i915_gem_object_create(obj);

225
	*handle_p = handle;
226
227
228
	return 0;
}

229
230
231
232
233
234
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
235
	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
	args->size = args->pitch * args->height;
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

int i915_gem_dumb_destroy(struct drm_file *file,
			  struct drm_device *dev,
			  uint32_t handle)
{
	return drm_gem_handle_delete(file, handle);
}

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
	struct drm_i915_gem_create *args = data;
256

257
258
259
260
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

261
static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
262
{
263
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
264
265

	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
266
		obj->tiling_mode != I915_TILING_NONE;
267
268
}

269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

295
static inline int
296
297
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

321
322
323
/* Per-page copy function for the shmem pread fastpath.
 * Flushes invalid cachelines before reading the target if
 * needs_clflush is set. */
324
static int
325
326
327
328
329
330
331
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

332
	if (unlikely(page_do_bit17_swizzling))
333
334
335
336
337
338
339
340
341
342
343
344
345
346
		return -EINVAL;

	vaddr = kmap_atomic(page);
	if (needs_clflush)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_to_user_inatomic(user_data,
				      vaddr + shmem_page_offset,
				      page_length);
	kunmap_atomic(vaddr);

	return ret;
}

347
348
349
350
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
351
	if (unlikely(swizzled)) {
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

369
370
371
372
373
374
375
376
377
378
379
380
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
381
382
383
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
384
385
386
387
388
389
390
391
392
393
394
395
396
397

	if (page_do_bit17_swizzling)
		ret = __copy_to_user_swizzled(user_data,
					      vaddr, shmem_page_offset,
					      page_length);
	else
		ret = __copy_to_user(user_data,
				     vaddr + shmem_page_offset,
				     page_length);
	kunmap(page);

	return ret;
}

398
static int
399
400
401
402
i915_gem_shmem_pread(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args,
		     struct drm_file *file)
403
{
404
	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
405
	char __user *user_data;
406
	ssize_t remain;
407
	loff_t offset;
408
	int shmem_page_offset, page_length, ret = 0;
409
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
410
	int hit_slowpath = 0;
411
	int prefaulted = 0;
412
	int needs_clflush = 0;
413
	int release_page;
414

415
	user_data = (char __user *) (uintptr_t) args->data_ptr;
416
417
	remain = args->size;

418
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
419

420
421
422
423
424
425
426
427
428
429
430
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		/* If we're not in the cpu read domain, set ourself into the gtt
		 * read domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will dirty the data
		 * anyway again before the next pread happens. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush = 1;
		ret = i915_gem_object_set_to_gtt_domain(obj, false);
		if (ret)
			return ret;
	}
431

432
	offset = args->offset;
433
434

	while (remain > 0) {
435
436
		struct page *page;

437
438
439
440
441
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
442
		shmem_page_offset = offset_in_page(offset);
443
444
445
446
		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

447
448
449
450
451
452
453
454
455
456
		if (obj->pages) {
			page = obj->pages[offset >> PAGE_SHIFT];
			release_page = 0;
		} else {
			page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
			if (IS_ERR(page)) {
				ret = PTR_ERR(page);
				goto out;
			}
			release_page = 1;
457
		}
458

459
460
461
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

462
463
464
465
466
		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
		if (ret == 0)
			goto next_page;
467
468

		hit_slowpath = 1;
469
		page_cache_get(page);
470
471
		mutex_unlock(&dev->struct_mutex);

472
		if (!prefaulted) {
473
			ret = fault_in_multipages_writeable(user_data, remain);
474
475
476
477
478
479
480
			/* Userspace is tricking us, but we've already clobbered
			 * its pages with the prefault and promised to write the
			 * data up to the first fault. Hence ignore any errors
			 * and just continue. */
			(void)ret;
			prefaulted = 1;
		}
481

482
483
484
		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
485

486
		mutex_lock(&dev->struct_mutex);
487
		page_cache_release(page);
488
next_page:
489
		mark_page_accessed(page);
490
491
		if (release_page)
			page_cache_release(page);
492

493
494
495
496
497
		if (ret) {
			ret = -EFAULT;
			goto out;
		}

498
		remain -= page_length;
499
		user_data += page_length;
500
501
502
		offset += page_length;
	}

503
out:
504
505
506
507
508
	if (hit_slowpath) {
		/* Fixup: Kill any reinstated backing storage pages */
		if (obj->madv == __I915_MADV_PURGED)
			i915_gem_object_truncate(obj);
	}
509
510
511
512

	return ret;
}

513
514
515
516
517
518
519
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
520
		     struct drm_file *file)
521
522
{
	struct drm_i915_gem_pread *args = data;
523
	struct drm_i915_gem_object *obj;
524
	int ret = 0;
525

526
527
528
529
530
531
532
533
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
		       (char __user *)(uintptr_t)args->data_ptr,
		       args->size))
		return -EFAULT;

534
	ret = i915_mutex_lock_interruptible(dev);
535
	if (ret)
536
		return ret;
537

538
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
539
	if (&obj->base == NULL) {
540
541
		ret = -ENOENT;
		goto unlock;
542
	}
543

544
	/* Bounds check source.  */
545
546
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
547
		ret = -EINVAL;
548
		goto out;
549
550
	}

551
552
553
554
555
556
557
558
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

Chris Wilson's avatar
Chris Wilson committed
559
560
	trace_i915_gem_object_pread(obj, args->offset, args->size);

561
	ret = i915_gem_shmem_pread(dev, obj, args, file);
562

563
out:
564
	drm_gem_object_unreference(&obj->base);
565
unlock:
566
	mutex_unlock(&dev->struct_mutex);
567
	return ret;
568
569
}

570
571
/* This is the fast write path which cannot handle
 * page faults in the source data
572
 */
573
574
575
576
577
578

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
579
{
580
581
	void __iomem *vaddr_atomic;
	void *vaddr;
582
	unsigned long unwritten;
583

Peter Zijlstra's avatar
Peter Zijlstra committed
584
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
585
586
587
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force*)vaddr_atomic + page_offset;
	unwritten = __copy_from_user_inatomic_nocache(vaddr,
588
						      user_data, length);
Peter Zijlstra's avatar
Peter Zijlstra committed
589
	io_mapping_unmap_atomic(vaddr_atomic);
590
	return unwritten;
591
592
}

593
594
595
596
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
597
static int
598
599
i915_gem_gtt_pwrite_fast(struct drm_device *dev,
			 struct drm_i915_gem_object *obj,
600
			 struct drm_i915_gem_pwrite *args,
601
			 struct drm_file *file)
602
{
603
	drm_i915_private_t *dev_priv = dev->dev_private;
604
	ssize_t remain;
605
	loff_t offset, page_base;
606
	char __user *user_data;
Daniel Vetter's avatar
Daniel Vetter committed
607
608
609
610
611
612
613
614
615
616
617
618
619
	int page_offset, page_length, ret;

	ret = i915_gem_object_pin(obj, 0, true);
	if (ret)
		goto out;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

	ret = i915_gem_object_put_fence(obj);
	if (ret)
		goto out_unpin;
620
621
622
623

	user_data = (char __user *) (uintptr_t) args->data_ptr;
	remain = args->size;

624
	offset = obj->gtt_offset + args->offset;
625
626
627
628

	while (remain > 0) {
		/* Operation in this page
		 *
629
630
631
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
632
		 */
633
634
		page_base = offset & PAGE_MASK;
		page_offset = offset_in_page(offset);
635
636
637
638
639
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		/* If we get a fault while copying data, then (presumably) our
640
641
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
642
		 */
643
		if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
Daniel Vetter's avatar
Daniel Vetter committed
644
645
646
647
				    page_offset, user_data, page_length)) {
			ret = -EFAULT;
			goto out_unpin;
		}
648

649
650
651
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
652
653
	}

Daniel Vetter's avatar
Daniel Vetter committed
654
655
656
out_unpin:
	i915_gem_object_unpin(obj);
out:
657
	return ret;
658
659
}

660
661
662
663
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set. */
664
static int
665
666
667
668
669
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
670
{
671
	char *vaddr;
672
	int ret;
673

674
	if (unlikely(page_do_bit17_swizzling))
675
		return -EINVAL;
676

677
678
679
680
681
682
683
684
685
686
687
	vaddr = kmap_atomic(page);
	if (needs_clflush_before)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
						user_data,
						page_length);
	if (needs_clflush_after)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	kunmap_atomic(vaddr);
688
689
690
691

	return ret;
}

692
693
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
694
static int
695
696
697
698
699
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
700
{
701
702
	char *vaddr;
	int ret;
703

704
	vaddr = kmap(page);
705
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
706
707
708
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
709
710
	if (page_do_bit17_swizzling)
		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
711
712
						user_data,
						page_length);
713
714
715
716
717
	else
		ret = __copy_from_user(vaddr + shmem_page_offset,
				       user_data,
				       page_length);
	if (needs_clflush_after)
718
719
720
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
721
	kunmap(page);
722

723
	return ret;
724
725
726
}

static int
727
728
729
730
i915_gem_shmem_pwrite(struct drm_device *dev,
		      struct drm_i915_gem_object *obj,
		      struct drm_i915_gem_pwrite *args,
		      struct drm_file *file)
731
{
732
	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
733
	ssize_t remain;
734
735
	loff_t offset;
	char __user *user_data;
736
	int shmem_page_offset, page_length, ret = 0;
737
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
738
	int hit_slowpath = 0;
739
740
	int needs_clflush_after = 0;
	int needs_clflush_before = 0;
741
	int release_page;
742

743
	user_data = (char __user *) (uintptr_t) args->data_ptr;
744
745
	remain = args->size;

746
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
747

748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		/* If we're not in the cpu write domain, set ourself into the gtt
		 * write domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will use the data
		 * right away and we therefore have to clflush anyway. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush_after = 1;
		ret = i915_gem_object_set_to_gtt_domain(obj, true);
		if (ret)
			return ret;
	}
	/* Same trick applies for invalidate partially written cachelines before
	 * writing.  */
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
	    && obj->cache_level == I915_CACHE_NONE)
		needs_clflush_before = 1;

765
	offset = args->offset;
766
	obj->dirty = 1;
767

768
	while (remain > 0) {
769
		struct page *page;
770
		int partial_cacheline_write;
771

772
773
774
775
776
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
777
		shmem_page_offset = offset_in_page(offset);
778
779
780
781
782

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

783
784
785
786
787
788
789
		/* If we don't overwrite a cacheline completely we need to be
		 * careful to have up-to-date data by first clflushing. Don't
		 * overcomplicate things and flush the entire patch. */
		partial_cacheline_write = needs_clflush_before &&
			((shmem_page_offset | page_length)
				& (boot_cpu_data.x86_clflush_size - 1));

790
791
792
793
794
795
796
797
798
799
		if (obj->pages) {
			page = obj->pages[offset >> PAGE_SHIFT];
			release_page = 0;
		} else {
			page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
			if (IS_ERR(page)) {
				ret = PTR_ERR(page);
				goto out;
			}
			release_page = 1;
800
801
		}

802
803
804
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

805
806
807
808
809
810
		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
		if (ret == 0)
			goto next_page;
811
812

		hit_slowpath = 1;
813
		page_cache_get(page);
814
815
		mutex_unlock(&dev->struct_mutex);

816
817
818
819
		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
820

821
		mutex_lock(&dev->struct_mutex);
822
		page_cache_release(page);
823
next_page:
824
825
		set_page_dirty(page);
		mark_page_accessed(page);
826
827
		if (release_page)
			page_cache_release(page);
828

829
830
831
832
833
		if (ret) {
			ret = -EFAULT;
			goto out;
		}

834
		remain -= page_length;
835
		user_data += page_length;
836
		offset += page_length;
837
838
	}

839
out:
840
841
842
843
844
845
846
847
848
849
	if (hit_slowpath) {
		/* Fixup: Kill any reinstated backing storage pages */
		if (obj->madv == __I915_MADV_PURGED)
			i915_gem_object_truncate(obj);
		/* and flush dirty cachelines in case the object isn't in the cpu write
		 * domain anymore. */
		if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
			i915_gem_clflush_object(obj);
			intel_gtt_chipset_flush();
		}
850
	}
851

852
853
854
	if (needs_clflush_after)
		intel_gtt_chipset_flush();

855
	return ret;
856
857
858
859
860
861
862
863
864
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
865
		      struct drm_file *file)
866
867
{
	struct drm_i915_gem_pwrite *args = data;
868
	struct drm_i915_gem_object *obj;
869
870
871
872
873
874
875
876
877
878
	int ret;

	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_READ,
		       (char __user *)(uintptr_t)args->data_ptr,
		       args->size))
		return -EFAULT;

879
880
	ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
					   args->size);
881
882
	if (ret)
		return -EFAULT;
883

884
	ret = i915_mutex_lock_interruptible(dev);
885
	if (ret)
886
		return ret;
887

888
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
889
	if (&obj->base == NULL) {
890
891
		ret = -ENOENT;
		goto unlock;
892
	}
893

894
	/* Bounds check destination. */
895
896
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
897
		ret = -EINVAL;
898
		goto out;
899
900
	}

901
902
903
904
905
906
907
908
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

Chris Wilson's avatar
Chris Wilson committed
909
910
	trace_i915_gem_object_pwrite(obj, args->offset, args->size);

Daniel Vetter's avatar
Daniel Vetter committed
911
	ret = -EFAULT;
912
913
914
915
916
917
	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
918
	if (obj->phys_obj) {
919
		ret = i915_gem_phys_pwrite(dev, obj, args, file);
920
921
922
923
		goto out;
	}

	if (obj->gtt_space &&
924
	    obj->cache_level == I915_CACHE_NONE &&
925
	    obj->tiling_mode == I915_TILING_NONE &&
926
	    obj->map_and_fenceable &&
927
	    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
928
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
Daniel Vetter's avatar
Daniel Vetter committed
929
930
931
		/* Note that the gtt paths might fail with non-page-backed user
		 * pointers (e.g. gtt mappings when moving data between
		 * textures). Fallback to the shmem path in that case. */
932
	}
933

934
	if (ret == -EFAULT)
Daniel Vetter's avatar
Daniel Vetter committed
935
		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
936

937
out:
938
	drm_gem_object_unreference(&obj->base);
939
unlock:
940
	mutex_unlock(&dev->struct_mutex);
941
942
943
944
	return ret;
}

/**
945
946
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
947
948
949
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
950
			  struct drm_file *file)
951
952
{
	struct drm_i915_gem_set_domain *args = data;
953
	struct drm_i915_gem_object *obj;
954
955
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
956
957
	int ret;

958
	/* Only handle setting domains to types used by the CPU. */
959
	if (write_domain & I915_GEM_GPU_DOMAINS)
960
961
		return -EINVAL;

962
	if (read_domains & I915_GEM_GPU_DOMAINS)
963
964
965
966
967
968
969
970
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

971
	ret = i915_mutex_lock_interruptible(dev);
972
	if (ret)
973
		return ret;
974

975
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
976
	if (&obj->base == NULL) {
977
978
		ret = -ENOENT;
		goto unlock;
979
	}
980

981
982
	if (read_domains & I915_GEM_DOMAIN_GTT) {
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
983
984
985
986
987
988
989

		/* Silently promote "you're not bound, there was nothing to do"
		 * to success, since the client was just asking us to
		 * make sure everything was done.
		 */
		if (ret == -EINVAL)
			ret = 0;
990
	} else {
991
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
992
993
	}

994
	drm_gem_object_unreference(&obj->base);
995
unlock:
996
997
998
999
1000
1001
1002
1003
1004
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1005
			 struct drm_file *file)
1006
1007
{
	struct drm_i915_gem_sw_finish *args = data;
1008
	struct drm_i915_gem_object *obj;
1009
1010
	int ret = 0;

1011
	ret = i915_mutex_lock_interruptible(dev);
1012
	if (ret)
1013
		return ret;
1014

1015
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1016
	if (&obj->base == NULL) {
1017
1018
		ret = -ENOENT;
		goto unlock;
1019
1020
1021
	}

	/* Pinned buffers may be scanout, so flush the cache */
1022
	if (obj->pin_count)
1023
1024
		i915_gem_object_flush_cpu_write_domain(obj);

1025
	drm_gem_object_unreference(&obj->base);
1026
unlock:
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1040
		    struct drm_file *file)
1041
1042
1043
1044
1045
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	unsigned long addr;

1046
	obj = drm_gem_object_lookup(dev, file, args->handle);
1047
	if (obj == NULL)
1048
		return -ENOENT;
1049

1050
1051
1052
1053
1054
1055
1056
1057
	/* prime objects have no backing filp to GEM mmap
	 * pages from.
	 */
	if (!obj->filp) {
		drm_gem_object_unreference_unlocked(obj);
		return -EINVAL;
	}

1058
	addr = vm_mmap(obj->filp, 0, args->size,
1059
1060
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
1061
	drm_gem_object_unreference_unlocked(obj);
1062
1063
1064
1065
1066
1067
1068
1069
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}