Commit b15ba512 authored by Jerome Glisse's avatar Jerome Glisse Committed by Dave Airlie
Browse files

drm/radeon: introduce a sub allocator and convert ib pool to it v4



Somewhat specializaed sub-allocator designed to perform sub-allocation
for command buffer not only for current cs ioctl but for future command
submission ioctl as well. Patch also convert current ib pool to use
the sub allocator. Idea is that ib poll buffer can be share with other
command buffer submission not having 64K granularity.

v2 Harmonize pool handling and add suspend/resume callback to pin/unpin
sa bo (tested on rv280, rv370, r420, rv515, rv610, rv710, redwood, cayman,
rs480, rs690, rs880)
v3 Simplify allocator
v4 Fix radeon_ib_get error path to properly free fence
Signed-off-by: default avatarJerome Glisse <jglisse@redhat.com>
Signed-off-by: default avatarDave Airlie <airlied@redhat.com>
parent 1b37078b
......@@ -71,7 +71,7 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
r600_blit_kms.o radeon_pm.o atombios_dp.o r600_audio.o r600_hdmi.o \
evergreen.o evergreen_cs.o evergreen_blit_shaders.o evergreen_blit_kms.o \
radeon_trace_points.o ni.o cayman_blit_shaders.o atombios_encoders.o \
radeon_semaphore.o
radeon_semaphore.o radeon_sa.o
radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
......
......@@ -3182,6 +3182,17 @@ static int evergreen_startup(struct radeon_device *rdev)
if (r)
return r;
r = radeon_ib_pool_start(rdev);
if (r)
return r;
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("radeon: failed testing IB (%d).\n", r);
rdev->accel_working = false;
return r;
}
return 0;
}
......@@ -3201,18 +3212,13 @@ int evergreen_resume(struct radeon_device *rdev)
/* post card */
atom_asic_init(rdev->mode_info.atom_context);
rdev->accel_working = true;
r = evergreen_startup(rdev);
if (r) {
DRM_ERROR("evergreen startup failed on resume\n");
return r;
}
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("radeon: failed testing IB (%d).\n", r);
return r;
}
return r;
}
......@@ -3222,12 +3228,13 @@ int evergreen_suspend(struct radeon_device *rdev)
struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
/* FIXME: we should wait for ring to be empty */
radeon_ib_pool_suspend(rdev);
r600_blit_suspend(rdev);
r700_cp_stop(rdev);
ring->ready = false;
evergreen_irq_suspend(rdev);
radeon_wb_disable(rdev);
evergreen_pcie_gart_disable(rdev);
r600_blit_suspend(rdev);
return 0;
}
......@@ -3312,29 +3319,24 @@ int evergreen_init(struct radeon_device *rdev)
if (r)
return r;
r = radeon_ib_pool_init(rdev);
rdev->accel_working = true;
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
rdev->accel_working = false;
}
r = evergreen_startup(rdev);
if (r) {
dev_err(rdev->dev, "disabling GPU acceleration\n");
r700_cp_fini(rdev);
r600_irq_fini(rdev);
radeon_wb_fini(rdev);
r100_ib_fini(rdev);
radeon_irq_kms_fini(rdev);
evergreen_pcie_gart_fini(rdev);
rdev->accel_working = false;
}
if (rdev->accel_working) {
r = radeon_ib_pool_init(rdev);
if (r) {
DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
rdev->accel_working = false;
}
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("radeon: failed testing IB (%d).\n", r);
rdev->accel_working = false;
}
}
return 0;
}
......@@ -3344,7 +3346,7 @@ void evergreen_fini(struct radeon_device *rdev)
r700_cp_fini(rdev);
r600_irq_fini(rdev);
radeon_wb_fini(rdev);
radeon_ib_pool_fini(rdev);
r100_ib_fini(rdev);
radeon_irq_kms_fini(rdev);
evergreen_pcie_gart_fini(rdev);
r600_vram_scratch_fini(rdev);
......
......@@ -1453,6 +1453,17 @@ static int cayman_startup(struct radeon_device *rdev)
if (r)
return r;
r = radeon_ib_pool_start(rdev);
if (r)
return r;
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("radeon: failed testing IB (%d).\n", r);
rdev->accel_working = false;
return r;
}
return 0;
}
......@@ -1467,32 +1478,25 @@ int cayman_resume(struct radeon_device *rdev)
/* post card */
atom_asic_init(rdev->mode_info.atom_context);
rdev->accel_working = true;
r = cayman_startup(rdev);
if (r) {
DRM_ERROR("cayman startup failed on resume\n");
return r;
}
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("radeon: failled testing IB (%d).\n", r);
return r;
}
return r;
}
int cayman_suspend(struct radeon_device *rdev)
{
/* FIXME: we should wait for ring to be empty */
radeon_ib_pool_suspend(rdev);
r600_blit_suspend(rdev);
cayman_cp_enable(rdev, false);
rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
evergreen_irq_suspend(rdev);
radeon_wb_disable(rdev);
cayman_pcie_gart_disable(rdev);
r600_blit_suspend(rdev);
return 0;
}
......@@ -1567,29 +1571,24 @@ int cayman_init(struct radeon_device *rdev)
if (r)
return r;
r = radeon_ib_pool_init(rdev);
rdev->accel_working = true;
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
rdev->accel_working = false;
}
r = cayman_startup(rdev);
if (r) {
dev_err(rdev->dev, "disabling GPU acceleration\n");
cayman_cp_fini(rdev);
r600_irq_fini(rdev);
radeon_wb_fini(rdev);
r100_ib_fini(rdev);
radeon_irq_kms_fini(rdev);
cayman_pcie_gart_fini(rdev);
rdev->accel_working = false;
}
if (rdev->accel_working) {
r = radeon_ib_pool_init(rdev);
if (r) {
DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
rdev->accel_working = false;
}
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("radeon: failed testing IB (%d).\n", r);
rdev->accel_working = false;
}
}
/* Don't start up if the MC ucode is missing.
* The default clocks and voltages before the MC ucode
......@@ -1609,7 +1608,7 @@ void cayman_fini(struct radeon_device *rdev)
cayman_cp_fini(rdev);
r600_irq_fini(rdev);
radeon_wb_fini(rdev);
radeon_ib_pool_fini(rdev);
r100_ib_fini(rdev);
radeon_irq_kms_fini(rdev);
cayman_pcie_gart_fini(rdev);
r600_vram_scratch_fini(rdev);
......
......@@ -3752,28 +3752,10 @@ int r100_ib_test(struct radeon_device *rdev)
void r100_ib_fini(struct radeon_device *rdev)
{
radeon_ib_pool_suspend(rdev);
radeon_ib_pool_fini(rdev);
}
int r100_ib_init(struct radeon_device *rdev)
{
int r;
r = radeon_ib_pool_init(rdev);
if (r) {
dev_err(rdev->dev, "failed initializing IB pool (%d).\n", r);
r100_ib_fini(rdev);
return r;
}
r = r100_ib_test(rdev);
if (r) {
dev_err(rdev->dev, "failed testing IB (%d).\n", r);
r100_ib_fini(rdev);
return r;
}
return 0;
}
void r100_mc_stop(struct radeon_device *rdev, struct r100_mc_save *save)
{
/* Shutdown CP we shouldn't need to do that but better be safe than
......@@ -3932,11 +3914,18 @@ static int r100_startup(struct radeon_device *rdev)
dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
return r;
}
r = r100_ib_init(rdev);
r = radeon_ib_pool_start(rdev);
if (r)
return r;
r = r100_ib_test(rdev);
if (r) {
dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
dev_err(rdev->dev, "failed testing IB (%d).\n", r);
rdev->accel_working = false;
return r;
}
return 0;
}
......@@ -3959,11 +3948,14 @@ int r100_resume(struct radeon_device *rdev)
r100_clock_startup(rdev);
/* Initialize surface registers */
radeon_surface_init(rdev);
rdev->accel_working = true;
return r100_startup(rdev);
}
int r100_suspend(struct radeon_device *rdev)
{
radeon_ib_pool_suspend(rdev);
r100_cp_disable(rdev);
radeon_wb_disable(rdev);
r100_irq_disable(rdev);
......@@ -4082,7 +4074,14 @@ int r100_init(struct radeon_device *rdev)
return r;
}
r100_set_safe_registers(rdev);
r = radeon_ib_pool_init(rdev);
rdev->accel_working = true;
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
rdev->accel_working = false;
}
r = r100_startup(rdev);
if (r) {
/* Somethings want wront with the accel init stop accel */
......
......@@ -1414,11 +1414,18 @@ static int r300_startup(struct radeon_device *rdev)
dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
return r;
}
r = r100_ib_init(rdev);
r = radeon_ib_pool_start(rdev);
if (r)
return r;
r = r100_ib_test(rdev);
if (r) {
dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
dev_err(rdev->dev, "failed testing IB (%d).\n", r);
rdev->accel_working = false;
return r;
}
return 0;
}
......@@ -1443,11 +1450,14 @@ int r300_resume(struct radeon_device *rdev)
r300_clock_startup(rdev);
/* Initialize surface registers */
radeon_surface_init(rdev);
rdev->accel_working = true;
return r300_startup(rdev);
}
int r300_suspend(struct radeon_device *rdev)
{
radeon_ib_pool_suspend(rdev);
r100_cp_disable(rdev);
radeon_wb_disable(rdev);
r100_irq_disable(rdev);
......@@ -1548,7 +1558,14 @@ int r300_init(struct radeon_device *rdev)
return r;
}
r300_set_reg_safe(rdev);
r = radeon_ib_pool_init(rdev);
rdev->accel_working = true;
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
rdev->accel_working = false;
}
r = r300_startup(rdev);
if (r) {
/* Somethings want wront with the accel init stop accel */
......
......@@ -274,11 +274,18 @@ static int r420_startup(struct radeon_device *rdev)
return r;
}
r420_cp_errata_init(rdev);
r = r100_ib_init(rdev);
r = radeon_ib_pool_start(rdev);
if (r)
return r;
r = r100_ib_test(rdev);
if (r) {
dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
dev_err(rdev->dev, "failed testing IB (%d).\n", r);
rdev->accel_working = false;
return r;
}
return 0;
}
......@@ -307,11 +314,14 @@ int r420_resume(struct radeon_device *rdev)
r420_clock_resume(rdev);
/* Initialize surface registers */
radeon_surface_init(rdev);
rdev->accel_working = true;
return r420_startup(rdev);
}
int r420_suspend(struct radeon_device *rdev)
{
radeon_ib_pool_suspend(rdev);
r420_cp_errata_fini(rdev);
r100_cp_disable(rdev);
radeon_wb_disable(rdev);
......@@ -424,7 +434,14 @@ int r420_init(struct radeon_device *rdev)
return r;
}
r420_set_reg_safe(rdev);
r = radeon_ib_pool_init(rdev);
rdev->accel_working = true;
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
rdev->accel_working = false;
}
r = r420_startup(rdev);
if (r) {
/* Somethings want wront with the accel init stop accel */
......
......@@ -202,9 +202,15 @@ static int r520_startup(struct radeon_device *rdev)
dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
return r;
}
r = r100_ib_init(rdev);
r = radeon_ib_pool_start(rdev);
if (r)
return r;
r = r100_ib_test(rdev);
if (r) {
dev_err(rdev->dev, "failed initializing IB (%d).\n", r);
dev_err(rdev->dev, "failed testing IB (%d).\n", r);
rdev->accel_working = false;
return r;
}
return 0;
......@@ -229,6 +235,8 @@ int r520_resume(struct radeon_device *rdev)
rv515_clock_startup(rdev);
/* Initialize surface registers */
radeon_surface_init(rdev);
rdev->accel_working = true;
return r520_startup(rdev);
}
......@@ -298,7 +306,14 @@ int r520_init(struct radeon_device *rdev)
if (r)
return r;
rv515_set_safe_registers(rdev);
r = radeon_ib_pool_init(rdev);
rdev->accel_working = true;
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
rdev->accel_working = false;
}
r = r520_startup(rdev);
if (r) {
/* Somethings want wront with the accel init stop accel */
......
......@@ -2486,6 +2486,17 @@ int r600_startup(struct radeon_device *rdev)
if (r)
return r;
r = radeon_ib_pool_start(rdev);
if (r)
return r;
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("radeon: failed testing IB (%d).\n", r);
rdev->accel_working = false;
return r;
}
return 0;
}
......@@ -2514,18 +2525,13 @@ int r600_resume(struct radeon_device *rdev)
/* post card */
atom_asic_init(rdev->mode_info.atom_context);
rdev->accel_working = true;
r = r600_startup(rdev);
if (r) {
DRM_ERROR("r600 startup failed on resume\n");
return r;
}
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
DRM_ERROR("radeon: failed testing IB (%d).\n", r);
return r;
}
r = r600_audio_init(rdev);
if (r) {
DRM_ERROR("radeon: audio resume failed\n");
......@@ -2538,13 +2544,14 @@ int r600_resume(struct radeon_device *rdev)
int r600_suspend(struct radeon_device *rdev)
{
r600_audio_fini(rdev);
radeon_ib_pool_suspend(rdev);
r600_blit_suspend(rdev);
/* FIXME: we should wait for ring to be empty */
r600_cp_stop(rdev);
rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
r600_irq_suspend(rdev);
radeon_wb_disable(rdev);
r600_pcie_gart_disable(rdev);
r600_blit_suspend(rdev);
return 0;
}
......@@ -2625,30 +2632,24 @@ int r600_init(struct radeon_device *rdev)
if (r)
return r;
r = radeon_ib_pool_init(rdev);
rdev->accel_working = true;
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
rdev->accel_working = false;
}
r = r600_startup(rdev);
if (r) {
dev_err(rdev->dev, "disabling GPU acceleration\n");
r600_cp_fini(rdev);
r600_irq_fini(rdev);
radeon_wb_fini(rdev);
r100_ib_fini(rdev);
radeon_irq_kms_fini(rdev);
r600_pcie_gart_fini(rdev);
rdev->accel_working = false;
}
if (rdev->accel_working) {
r = radeon_ib_pool_init(rdev);
if (r) {
dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
rdev->accel_working = false;
} else {
r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
if (r) {
dev_err(rdev->dev, "IB test failed (%d).\n", r);
rdev->accel_working = false;
}
}
}
r = r600_audio_init(rdev);
if (r)
......@@ -2663,7 +2664,7 @@ void r600_fini(struct radeon_device *rdev)
r600_cp_fini(rdev);
r600_irq_fini(rdev);
radeon_wb_fini(rdev);
radeon_ib_pool_fini(rdev);
r100_ib_fini(rdev);
radeon_irq_kms_fini(rdev);
r600_pcie_gart_fini(rdev);
r600_vram_scratch_fini(rdev);
......
......@@ -316,6 +316,48 @@ struct radeon_bo_list {
u32 tiling_flags;
};
/* sub-allocation manager, it has to be protected by another lock.
* By conception this is an helper for other part of the driver
* like the indirect buffer or semaphore, which both have their
* locking.
*
* Principe is simple, we keep a list of sub allocation in offset
* order (first entry has offset == 0, last entry has the highest
* offset).
*
* When allocating new object we first check if there is room at
* the end total_size - (last_object_offset + last_object_size) >=
* alloc_size. If so we allocate new object there.
*
* When there is not enough room at the end, we start waiting for
* each sub object until we reach object_offset+object_size >=
* alloc_size, this object then become the sub object we return.
*
* Alignment can't be bigger than page size.
*
* Hole are not considered for allocation to keep things simple.
* Assumption is that there won't be hole (all object on same
* alignment).
*/
struct radeon_sa_manager {
struct radeon_bo *bo;
struct list_head sa_bo;
unsigned size;
uint64_t gpu_addr;
void *cpu_ptr;
uint32_t domain;
};
struct radeon_sa_bo;
/* sub-allocation buffer */
struct radeon_sa_bo {
struct list_head list;
struct radeon_sa_manager *manager;
unsigned offset;
unsigned size;
};
/*
* GEM objects.
*/
......@@ -503,13 +545,12 @@ void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
*/
struct radeon_ib {
struct list_head list;
struct radeon_sa_bo sa_bo;
unsigned idx;
uint32_t length_dw;
uint64_t gpu_addr;
struct radeon_fence *fence;
uint32_t *ptr;
uint32_t length_dw;
bool free;
struct radeon_fence *fence;
};
/*
......@@ -517,12 +558,11 @@ struct radeon_ib {
* mutex protects scheduled_ibs, ready, alloc_bm
*/
struct radeon_ib_pool {
struct mutex mutex;
struct radeon_bo *robj;
struct list_head bogus_ib;
struct radeon_ib ibs[RADEON_IB_POOL_SIZE];
bool ready;
unsigned head_id;
struct mutex mutex;
struct radeon_sa_manager sa_manager;
struct radeon_ib ibs[RADEON_IB_POOL_SIZE];
bool ready;
unsigned head_id;
};
struct radeon_ring {
......@@ -603,8 +643,9 @@ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib);
int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib);
int radeon_ib_pool_init(struct radeon_device *rdev);
void radeon_ib_pool_fini(struct radeon_device *rdev);
int radeon_ib_pool_start(struct radeon_device *rdev);
int radeon_ib_pool_suspend(struct radeon_device *rdev);
int radeon_ib_test(struct radeon_device *rdev);