All new accounts created on Gitlab now require administrator approval. If you invite any collaborators, please let Flux staff know so they can approve the accounts.

drm_gem_cma_helper.c 15.2 KB
Newer Older
Sascha Hauer's avatar
Sascha Hauer committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * drm gem CMA (contiguous memory allocator) helper functions
 *
 * Copyright (C) 2012 Sascha Hauer, Pengutronix
 *
 * Based on Samsung Exynos code
 *
 * Copyright (c) 2011 Samsung Electronics Co., Ltd.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/mutex.h>
#include <linux/export.h>
24
#include <linux/dma-buf.h>
Sascha Hauer's avatar
Sascha Hauer committed
25 26 27 28 29
#include <linux/dma-mapping.h>

#include <drm/drmP.h>
#include <drm/drm.h>
#include <drm/drm_gem_cma_helper.h>
30
#include <drm/drm_vma_manager.h>
Sascha Hauer's avatar
Sascha Hauer committed
31

32 33 34 35 36 37 38 39 40 41 42 43
/**
 * DOC: cma helpers
 *
 * The Contiguous Memory Allocator reserves a pool of memory at early boot
 * that is used to service requests for large blocks of contiguous memory.
 *
 * The DRM GEM/CMA helpers use this allocator as a means to provide buffer
 * objects that are physically contiguous in memory. This is useful for
 * display drivers that are unable to map scattered buffers via an IOMMU.
 */

/**
44
 * __drm_gem_cma_create - Create a GEM CMA object without allocating memory
45 46
 * @drm: DRM device
 * @size: size of the object to allocate
Sascha Hauer's avatar
Sascha Hauer committed
47
 *
48 49
 * This function creates and initializes a GEM CMA object of the given size,
 * but doesn't allocate any memory to back the object.
50
 *
51 52 53
 * Returns:
 * A struct drm_gem_cma_object * on success or an ERR_PTR()-encoded negative
 * error code on failure.
Sascha Hauer's avatar
Sascha Hauer committed
54
 */
55
static struct drm_gem_cma_object *
56
__drm_gem_cma_create(struct drm_device *drm, size_t size)
Sascha Hauer's avatar
Sascha Hauer committed
57 58 59 60 61
{
	struct drm_gem_cma_object *cma_obj;
	struct drm_gem_object *gem_obj;
	int ret;

62 63 64 65 66
	if (drm->driver->gem_create_object)
		gem_obj = drm->driver->gem_create_object(drm, size);
	else
		gem_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
	if (!gem_obj)
Sascha Hauer's avatar
Sascha Hauer committed
67
		return ERR_PTR(-ENOMEM);
68
	cma_obj = container_of(gem_obj, struct drm_gem_cma_object, base);
Sascha Hauer's avatar
Sascha Hauer committed
69 70 71

	ret = drm_gem_object_init(drm, gem_obj, size);
	if (ret)
72
		goto error;
Sascha Hauer's avatar
Sascha Hauer committed
73 74

	ret = drm_gem_create_mmap_offset(gem_obj);
75 76 77 78
	if (ret) {
		drm_gem_object_release(gem_obj);
		goto error;
	}
Sascha Hauer's avatar
Sascha Hauer committed
79 80 81

	return cma_obj;

82 83 84 85
error:
	kfree(cma_obj);
	return ERR_PTR(ret);
}
Sascha Hauer's avatar
Sascha Hauer committed
86

87
/**
88
 * drm_gem_cma_create - allocate an object with the given size
89 90 91 92 93 94
 * @drm: DRM device
 * @size: size of the object to allocate
 *
 * This function creates a CMA GEM object and allocates a contiguous chunk of
 * memory as backing store. The backing memory has the writecombine attribute
 * set.
95
 *
96 97 98
 * Returns:
 * A struct drm_gem_cma_object * on success or an ERR_PTR()-encoded negative
 * error code on failure.
99 100
 */
struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
101
					      size_t size)
102 103
{
	struct drm_gem_cma_object *cma_obj;
104
	int ret;
Sascha Hauer's avatar
Sascha Hauer committed
105

106
	size = round_up(size, PAGE_SIZE);
Sascha Hauer's avatar
Sascha Hauer committed
107

108 109 110 111
	cma_obj = __drm_gem_cma_create(drm, size);
	if (IS_ERR(cma_obj))
		return cma_obj;

112 113
	cma_obj->vaddr = dma_alloc_wc(drm->dev, size, &cma_obj->paddr,
				      GFP_KERNEL | __GFP_NOWARN);
114
	if (!cma_obj->vaddr) {
115
		dev_err(drm->dev, "failed to allocate buffer with size %zu\n",
116
			size);
117 118 119 120
		ret = -ENOMEM;
		goto error;
	}

121
	return cma_obj;
122 123

error:
124
	drm_gem_object_unreference_unlocked(&cma_obj->base);
125
	return ERR_PTR(ret);
Sascha Hauer's avatar
Sascha Hauer committed
126 127 128
}
EXPORT_SYMBOL_GPL(drm_gem_cma_create);

129 130 131 132 133 134 135 136 137 138 139
/**
 * drm_gem_cma_create_with_handle - allocate an object with the given size and
 *     return a GEM handle to it
 * @file_priv: DRM file-private structure to register the handle for
 * @drm: DRM device
 * @size: size of the object to allocate
 * @handle: return location for the GEM handle
 *
 * This function creates a CMA GEM object, allocating a physically contiguous
 * chunk of memory as backing store. The GEM object is then added to the list
 * of object associated with the given file and a handle to it is returned.
Sascha Hauer's avatar
Sascha Hauer committed
140
 *
141 142 143
 * Returns:
 * A struct drm_gem_cma_object * on success or an ERR_PTR()-encoded negative
 * error code on failure.
Sascha Hauer's avatar
Sascha Hauer committed
144
 */
145 146 147 148
static struct drm_gem_cma_object *
drm_gem_cma_create_with_handle(struct drm_file *file_priv,
			       struct drm_device *drm, size_t size,
			       uint32_t *handle)
Sascha Hauer's avatar
Sascha Hauer committed
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
{
	struct drm_gem_cma_object *cma_obj;
	struct drm_gem_object *gem_obj;
	int ret;

	cma_obj = drm_gem_cma_create(drm, size);
	if (IS_ERR(cma_obj))
		return cma_obj;

	gem_obj = &cma_obj->base;

	/*
	 * allocate a id of idr table where the obj is registered
	 * and handle has the id what user can see.
	 */
	ret = drm_gem_handle_create(file_priv, gem_obj, handle);
	/* drop reference from allocate - handle holds it now. */
	drm_gem_object_unreference_unlocked(gem_obj);
167 168
	if (ret)
		return ERR_PTR(ret);
Sascha Hauer's avatar
Sascha Hauer committed
169 170 171 172

	return cma_obj;
}

173 174 175 176 177 178 179 180
/**
 * drm_gem_cma_free_object - free resources associated with a CMA GEM object
 * @gem_obj: GEM object to free
 *
 * This function frees the backing memory of the CMA GEM object, cleans up the
 * GEM object state and frees the memory used to store the object itself.
 * Drivers using the CMA helpers should set this as their DRM driver's
 * ->gem_free_object() callback.
Sascha Hauer's avatar
Sascha Hauer committed
181 182 183 184 185 186 187
 */
void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
{
	struct drm_gem_cma_object *cma_obj;

	cma_obj = to_drm_gem_cma_obj(gem_obj);

188
	if (cma_obj->vaddr) {
189 190
		dma_free_wc(gem_obj->dev->dev, cma_obj->base.size,
			    cma_obj->vaddr, cma_obj->paddr);
191 192 193
	} else if (gem_obj->import_attach) {
		drm_prime_gem_destroy(gem_obj, cma_obj->sgt);
	}
194 195

	drm_gem_object_release(gem_obj);
Sascha Hauer's avatar
Sascha Hauer committed
196 197 198 199 200

	kfree(cma_obj);
}
EXPORT_SYMBOL_GPL(drm_gem_cma_free_object);

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
/**
 * drm_gem_cma_dumb_create_internal - create a dumb buffer object
 * @file_priv: DRM file-private structure to create the dumb buffer for
 * @drm: DRM device
 * @args: IOCTL data
 *
 * This aligns the pitch and size arguments to the minimum required. This is
 * an internal helper that can be wrapped by a driver to account for hardware
 * with more specific alignment requirements. It should not be used directly
 * as the ->dumb_create() callback in a DRM driver.
 *
 * Returns:
 * 0 on success or a negative error code on failure.
 */
int drm_gem_cma_dumb_create_internal(struct drm_file *file_priv,
				     struct drm_device *drm,
				     struct drm_mode_create_dumb *args)
{
	unsigned int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
	struct drm_gem_cma_object *cma_obj;

	if (args->pitch < min_pitch)
		args->pitch = min_pitch;

	if (args->size < args->pitch * args->height)
		args->size = args->pitch * args->height;

	cma_obj = drm_gem_cma_create_with_handle(file_priv, drm, args->size,
						 &args->handle);
	return PTR_ERR_OR_ZERO(cma_obj);
}
EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create_internal);

234 235 236 237 238 239 240 241 242 243
/**
 * drm_gem_cma_dumb_create - create a dumb buffer object
 * @file_priv: DRM file-private structure to create the dumb buffer for
 * @drm: DRM device
 * @args: IOCTL data
 *
 * This function computes the pitch of the dumb buffer and rounds it up to an
 * integer number of bytes per pixel. Drivers for hardware that doesn't have
 * any additional restrictions on the pitch can directly use this function as
 * their ->dumb_create() callback.
Sascha Hauer's avatar
Sascha Hauer committed
244
 *
245 246 247 248
 * For hardware with additional restrictions, drivers can adjust the fields
 * set up by userspace and pass the IOCTL data along to the
 * drm_gem_cma_dumb_create_internal() function.
 *
249 250
 * Returns:
 * 0 on success or a negative error code on failure.
Sascha Hauer's avatar
Sascha Hauer committed
251 252
 */
int drm_gem_cma_dumb_create(struct drm_file *file_priv,
253 254
			    struct drm_device *drm,
			    struct drm_mode_create_dumb *args)
Sascha Hauer's avatar
Sascha Hauer committed
255 256 257
{
	struct drm_gem_cma_object *cma_obj;

258 259
	args->pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
	args->size = args->pitch * args->height;
Sascha Hauer's avatar
Sascha Hauer committed
260

261 262
	cma_obj = drm_gem_cma_create_with_handle(file_priv, drm, args->size,
						 &args->handle);
263
	return PTR_ERR_OR_ZERO(cma_obj);
Sascha Hauer's avatar
Sascha Hauer committed
264 265 266
}
EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create);

267 268 269 270 271 272 273 274 275 276 277 278 279 280
/**
 * drm_gem_cma_dumb_map_offset - return the fake mmap offset for a CMA GEM
 *     object
 * @file_priv: DRM file-private structure containing the GEM object
 * @drm: DRM device
 * @handle: GEM object handle
 * @offset: return location for the fake mmap offset
 *
 * This function look up an object by its handle and returns the fake mmap
 * offset associated with it. Drivers using the CMA helpers should set this
 * as their DRM driver's ->dumb_map_offset() callback.
 *
 * Returns:
 * 0 on success or a negative error code on failure.
Sascha Hauer's avatar
Sascha Hauer committed
281 282
 */
int drm_gem_cma_dumb_map_offset(struct drm_file *file_priv,
283 284
				struct drm_device *drm, u32 handle,
				u64 *offset)
Sascha Hauer's avatar
Sascha Hauer committed
285 286 287
{
	struct drm_gem_object *gem_obj;

288
	gem_obj = drm_gem_object_lookup(file_priv, handle);
Sascha Hauer's avatar
Sascha Hauer committed
289
	if (!gem_obj) {
290
		dev_err(drm->dev, "failed to lookup GEM object\n");
Sascha Hauer's avatar
Sascha Hauer committed
291 292 293
		return -EINVAL;
	}

294
	*offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
Sascha Hauer's avatar
Sascha Hauer committed
295

296
	drm_gem_object_unreference_unlocked(gem_obj);
Sascha Hauer's avatar
Sascha Hauer committed
297 298 299 300 301 302 303 304 305 306 307

	return 0;
}
EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_map_offset);

const struct vm_operations_struct drm_gem_cma_vm_ops = {
	.open = drm_gem_vm_open,
	.close = drm_gem_vm_close,
};
EXPORT_SYMBOL_GPL(drm_gem_cma_vm_ops);

308 309 310 311 312
static int drm_gem_cma_mmap_obj(struct drm_gem_cma_object *cma_obj,
				struct vm_area_struct *vma)
{
	int ret;

313 314 315 316 317 318 319 320
	/*
	 * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
	 * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
	 * the whole buffer.
	 */
	vma->vm_flags &= ~VM_PFNMAP;
	vma->vm_pgoff = 0;

321 322
	ret = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr,
			  cma_obj->paddr, vma->vm_end - vma->vm_start);
323 324 325 326 327 328
	if (ret)
		drm_gem_vm_close(vma);

	return ret;
}

329 330 331 332 333 334 335 336 337 338 339 340 341 342
/**
 * drm_gem_cma_mmap - memory-map a CMA GEM object
 * @filp: file object
 * @vma: VMA for the area to be mapped
 *
 * This function implements an augmented version of the GEM DRM file mmap
 * operation for CMA objects: In addition to the usual GEM VMA setup it
 * immediately faults in the entire object instead of using on-demaind
 * faulting. Drivers which employ the CMA helpers should use this function
 * as their ->mmap() handler in the DRM device file's file_operations
 * structure.
 *
 * Returns:
 * 0 on success or a negative error code on failure.
Sascha Hauer's avatar
Sascha Hauer committed
343 344 345 346
 */
int drm_gem_cma_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct drm_gem_cma_object *cma_obj;
347
	struct drm_gem_object *gem_obj;
Sascha Hauer's avatar
Sascha Hauer committed
348 349 350 351 352 353 354 355 356
	int ret;

	ret = drm_gem_mmap(filp, vma);
	if (ret)
		return ret;

	gem_obj = vma->vm_private_data;
	cma_obj = to_drm_gem_cma_obj(gem_obj);

357
	return drm_gem_cma_mmap_obj(cma_obj, vma);
Sascha Hauer's avatar
Sascha Hauer committed
358 359 360
}
EXPORT_SYMBOL_GPL(drm_gem_cma_mmap);

Rob Clark's avatar
Rob Clark committed
361
#ifdef CONFIG_DEBUG_FS
362 363 364 365 366 367 368 369 370 371
/**
 * drm_gem_cma_describe - describe a CMA GEM object for debugfs
 * @cma_obj: CMA GEM object
 * @m: debugfs file handle
 *
 * This function can be used to dump a human-readable representation of the
 * CMA GEM object into a synthetic file.
 */
void drm_gem_cma_describe(struct drm_gem_cma_object *cma_obj,
			  struct seq_file *m)
Rob Clark's avatar
Rob Clark committed
372 373
{
	struct drm_gem_object *obj = &cma_obj->base;
374
	uint64_t off;
Rob Clark's avatar
Rob Clark committed
375

376
	off = drm_vma_node_start(&obj->vma_node);
Rob Clark's avatar
Rob Clark committed
377

378
	seq_printf(m, "%2d (%2d) %08llx %pad %p %zu",
Rob Clark's avatar
Rob Clark committed
379
			obj->name, obj->refcount.refcount.counter,
380
			off, &cma_obj->paddr, cma_obj->vaddr, obj->size);
Rob Clark's avatar
Rob Clark committed
381 382 383 384 385

	seq_printf(m, "\n");
}
EXPORT_SYMBOL_GPL(drm_gem_cma_describe);
#endif
386

387 388 389 390 391 392 393 394 395 396 397 398
/**
 * drm_gem_cma_prime_get_sg_table - provide a scatter/gather table of pinned
 *     pages for a CMA GEM object
 * @obj: GEM object
 *
 * This function exports a scatter/gather table suitable for PRIME usage by
 * calling the standard DMA mapping API. Drivers using the CMA helpers should
 * set this as their DRM driver's ->gem_prime_get_sg_table() callback.
 *
 * Returns:
 * A pointer to the scatter/gather table of pinned pages or NULL on failure.
 */
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
struct sg_table *drm_gem_cma_prime_get_sg_table(struct drm_gem_object *obj)
{
	struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
	struct sg_table *sgt;
	int ret;

	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
	if (!sgt)
		return NULL;

	ret = dma_get_sgtable(obj->dev->dev, sgt, cma_obj->vaddr,
			      cma_obj->paddr, obj->size);
	if (ret < 0)
		goto out;

	return sgt;

out:
	kfree(sgt);
	return NULL;
}
EXPORT_SYMBOL_GPL(drm_gem_cma_prime_get_sg_table);

422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
/**
 * drm_gem_cma_prime_import_sg_table - produce a CMA GEM object from another
 *     driver's scatter/gather table of pinned pages
 * @dev: device to import into
 * @attach: DMA-BUF attachment
 * @sgt: scatter/gather table of pinned pages
 *
 * This function imports a scatter/gather table exported via DMA-BUF by
 * another driver. Imported buffers must be physically contiguous in memory
 * (i.e. the scatter/gather table must contain a single entry). Drivers that
 * use the CMA helpers should set this as their DRM driver's
 * ->gem_prime_import_sg_table() callback.
 *
 * Returns:
 * A pointer to a newly created GEM object or an ERR_PTR-encoded negative
 * error code on failure.
 */
439
struct drm_gem_object *
440 441
drm_gem_cma_prime_import_sg_table(struct drm_device *dev,
				  struct dma_buf_attachment *attach,
442 443 444 445 446 447 448 449
				  struct sg_table *sgt)
{
	struct drm_gem_cma_object *cma_obj;

	if (sgt->nents != 1)
		return ERR_PTR(-EINVAL);

	/* Create a CMA GEM buffer. */
450
	cma_obj = __drm_gem_cma_create(dev, attach->dmabuf->size);
451
	if (IS_ERR(cma_obj))
452
		return ERR_CAST(cma_obj);
453 454 455 456

	cma_obj->paddr = sg_dma_address(sgt->sgl);
	cma_obj->sgt = sgt;

457
	DRM_DEBUG_PRIME("dma_addr = %pad, size = %zu\n", &cma_obj->paddr, attach->dmabuf->size);
458 459 460 461 462

	return &cma_obj->base;
}
EXPORT_SYMBOL_GPL(drm_gem_cma_prime_import_sg_table);

463 464 465 466 467 468 469 470 471 472 473 474
/**
 * drm_gem_cma_prime_mmap - memory-map an exported CMA GEM object
 * @obj: GEM object
 * @vma: VMA for the area to be mapped
 *
 * This function maps a buffer imported via DRM PRIME into a userspace
 * process's address space. Drivers that use the CMA helpers should set this
 * as their DRM driver's ->gem_prime_mmap() callback.
 *
 * Returns:
 * 0 on success or a negative error code on failure.
 */
475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
int drm_gem_cma_prime_mmap(struct drm_gem_object *obj,
			   struct vm_area_struct *vma)
{
	struct drm_gem_cma_object *cma_obj;
	int ret;

	ret = drm_gem_mmap_obj(obj, obj->size, vma);
	if (ret < 0)
		return ret;

	cma_obj = to_drm_gem_cma_obj(obj);
	return drm_gem_cma_mmap_obj(cma_obj, vma);
}
EXPORT_SYMBOL_GPL(drm_gem_cma_prime_mmap);

490 491 492 493 494 495 496 497 498 499 500 501 502 503
/**
 * drm_gem_cma_prime_vmap - map a CMA GEM object into the kernel's virtual
 *     address space
 * @obj: GEM object
 *
 * This function maps a buffer exported via DRM PRIME into the kernel's
 * virtual address space. Since the CMA buffers are already mapped into the
 * kernel virtual address space this simply returns the cached virtual
 * address. Drivers using the CMA helpers should set this as their DRM
 * driver's ->gem_prime_vmap() callback.
 *
 * Returns:
 * The kernel virtual address of the CMA GEM object's backing store.
 */
504 505 506 507 508 509 510 511
void *drm_gem_cma_prime_vmap(struct drm_gem_object *obj)
{
	struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);

	return cma_obj->vaddr;
}
EXPORT_SYMBOL_GPL(drm_gem_cma_prime_vmap);

512 513 514 515 516 517 518 519 520 521 522
/**
 * drm_gem_cma_prime_vunmap - unmap a CMA GEM object from the kernel's virtual
 *     address space
 * @obj: GEM object
 * @vaddr: kernel virtual address where the CMA GEM object was mapped
 *
 * This function removes a buffer exported via DRM PRIME from the kernel's
 * virtual address space. This is a no-op because CMA buffers cannot be
 * unmapped from kernel space. Drivers using the CMA helpers should set this
 * as their DRM driver's ->gem_prime_vunmap() callback.
 */
523 524 525 526 527
void drm_gem_cma_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
{
	/* Nothing to do */
}
EXPORT_SYMBOL_GPL(drm_gem_cma_prime_vunmap);