mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-23 18:25:51 +07:00
a266bf4200
In order to avoid some nasty mutex inversions, commit09c5ab384f
("drm/i915: Keep rings pinned while the context is active") allowed the intel_ring unpinning to be run concurrently with the next context pinning it. Thus each step in intel_ring_unpin() needed to be atomic and ordered in a nice onion with intel_ring_pin() so that the lifetimes overlapped and were always safe. Sadly, a few steps in intel_ring_unpin() were overlooked, such as closing the read/write pointers of the ring and discarding the intel_ring.vaddr, as these steps were not serialised with intel_ring_pin() and so could leave the ring in disarray. Fixes:09c5ab384f
("drm/i915: Keep rings pinned while the context is active") Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> Cc: Mika Kuoppala <mika.kuoppala@linux.intel.com> Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com> Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com> Link: https://patchwork.freedesktop.org/patch/msgid/20191118230254.2615942-6-chris@chris-wilson.co.uk
319 lines
7.2 KiB
C
319 lines
7.2 KiB
C
/*
|
|
* SPDX-License-Identifier: MIT
|
|
*
|
|
* Copyright © 2019 Intel Corporation
|
|
*/
|
|
|
|
#include "gem/i915_gem_object.h"
|
|
#include "i915_drv.h"
|
|
#include "i915_vma.h"
|
|
#include "intel_engine.h"
|
|
#include "intel_ring.h"
|
|
#include "intel_timeline.h"
|
|
|
|
unsigned int intel_ring_update_space(struct intel_ring *ring)
|
|
{
|
|
unsigned int space;
|
|
|
|
space = __intel_ring_space(ring->head, ring->emit, ring->size);
|
|
|
|
ring->space = space;
|
|
return space;
|
|
}
|
|
|
|
int intel_ring_pin(struct intel_ring *ring)
|
|
{
|
|
struct i915_vma *vma = ring->vma;
|
|
unsigned int flags;
|
|
void *addr;
|
|
int ret;
|
|
|
|
if (atomic_fetch_inc(&ring->pin_count))
|
|
return 0;
|
|
|
|
flags = PIN_GLOBAL;
|
|
|
|
/* Ring wraparound at offset 0 sometimes hangs. No idea why. */
|
|
flags |= PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
|
|
|
|
if (vma->obj->stolen)
|
|
flags |= PIN_MAPPABLE;
|
|
else
|
|
flags |= PIN_HIGH;
|
|
|
|
ret = i915_vma_pin(vma, 0, 0, flags);
|
|
if (unlikely(ret))
|
|
goto err_unpin;
|
|
|
|
if (i915_vma_is_map_and_fenceable(vma))
|
|
addr = (void __force *)i915_vma_pin_iomap(vma);
|
|
else
|
|
addr = i915_gem_object_pin_map(vma->obj,
|
|
i915_coherent_map_type(vma->vm->i915));
|
|
if (IS_ERR(addr)) {
|
|
ret = PTR_ERR(addr);
|
|
goto err_ring;
|
|
}
|
|
|
|
i915_vma_make_unshrinkable(vma);
|
|
|
|
/* Discard any unused bytes beyond that submitted to hw. */
|
|
intel_ring_reset(ring, ring->emit);
|
|
|
|
ring->vaddr = addr;
|
|
return 0;
|
|
|
|
err_ring:
|
|
i915_vma_unpin(vma);
|
|
err_unpin:
|
|
atomic_dec(&ring->pin_count);
|
|
return ret;
|
|
}
|
|
|
|
void intel_ring_reset(struct intel_ring *ring, u32 tail)
|
|
{
|
|
tail = intel_ring_wrap(ring, tail);
|
|
ring->tail = tail;
|
|
ring->head = tail;
|
|
ring->emit = tail;
|
|
intel_ring_update_space(ring);
|
|
}
|
|
|
|
void intel_ring_unpin(struct intel_ring *ring)
|
|
{
|
|
struct i915_vma *vma = ring->vma;
|
|
|
|
if (!atomic_dec_and_test(&ring->pin_count))
|
|
return;
|
|
|
|
i915_vma_unset_ggtt_write(vma);
|
|
if (i915_vma_is_map_and_fenceable(vma))
|
|
i915_vma_unpin_iomap(vma);
|
|
else
|
|
i915_gem_object_unpin_map(vma->obj);
|
|
|
|
i915_vma_make_purgeable(vma);
|
|
i915_vma_unpin(vma);
|
|
}
|
|
|
|
static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)
|
|
{
|
|
struct i915_address_space *vm = &ggtt->vm;
|
|
struct drm_i915_private *i915 = vm->i915;
|
|
struct drm_i915_gem_object *obj;
|
|
struct i915_vma *vma;
|
|
|
|
obj = ERR_PTR(-ENODEV);
|
|
if (i915_ggtt_has_aperture(ggtt))
|
|
obj = i915_gem_object_create_stolen(i915, size);
|
|
if (IS_ERR(obj))
|
|
obj = i915_gem_object_create_internal(i915, size);
|
|
if (IS_ERR(obj))
|
|
return ERR_CAST(obj);
|
|
|
|
/*
|
|
* Mark ring buffers as read-only from GPU side (so no stray overwrites)
|
|
* if supported by the platform's GGTT.
|
|
*/
|
|
if (vm->has_read_only)
|
|
i915_gem_object_set_readonly(obj);
|
|
|
|
vma = i915_vma_instance(obj, vm, NULL);
|
|
if (IS_ERR(vma))
|
|
goto err;
|
|
|
|
return vma;
|
|
|
|
err:
|
|
i915_gem_object_put(obj);
|
|
return vma;
|
|
}
|
|
|
|
struct intel_ring *
|
|
intel_engine_create_ring(struct intel_engine_cs *engine, int size)
|
|
{
|
|
struct drm_i915_private *i915 = engine->i915;
|
|
struct intel_ring *ring;
|
|
struct i915_vma *vma;
|
|
|
|
GEM_BUG_ON(!is_power_of_2(size));
|
|
GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
|
|
|
|
ring = kzalloc(sizeof(*ring), GFP_KERNEL);
|
|
if (!ring)
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
kref_init(&ring->ref);
|
|
ring->size = size;
|
|
|
|
/*
|
|
* Workaround an erratum on the i830 which causes a hang if
|
|
* the TAIL pointer points to within the last 2 cachelines
|
|
* of the buffer.
|
|
*/
|
|
ring->effective_size = size;
|
|
if (IS_I830(i915) || IS_I845G(i915))
|
|
ring->effective_size -= 2 * CACHELINE_BYTES;
|
|
|
|
intel_ring_update_space(ring);
|
|
|
|
vma = create_ring_vma(engine->gt->ggtt, size);
|
|
if (IS_ERR(vma)) {
|
|
kfree(ring);
|
|
return ERR_CAST(vma);
|
|
}
|
|
ring->vma = vma;
|
|
|
|
return ring;
|
|
}
|
|
|
|
void intel_ring_free(struct kref *ref)
|
|
{
|
|
struct intel_ring *ring = container_of(ref, typeof(*ring), ref);
|
|
|
|
i915_vma_put(ring->vma);
|
|
kfree(ring);
|
|
}
|
|
|
|
static noinline int
|
|
wait_for_space(struct intel_ring *ring,
|
|
struct intel_timeline *tl,
|
|
unsigned int bytes)
|
|
{
|
|
struct i915_request *target;
|
|
long timeout;
|
|
|
|
if (intel_ring_update_space(ring) >= bytes)
|
|
return 0;
|
|
|
|
GEM_BUG_ON(list_empty(&tl->requests));
|
|
list_for_each_entry(target, &tl->requests, link) {
|
|
if (target->ring != ring)
|
|
continue;
|
|
|
|
/* Would completion of this request free enough space? */
|
|
if (bytes <= __intel_ring_space(target->postfix,
|
|
ring->emit, ring->size))
|
|
break;
|
|
}
|
|
|
|
if (GEM_WARN_ON(&target->link == &tl->requests))
|
|
return -ENOSPC;
|
|
|
|
timeout = i915_request_wait(target,
|
|
I915_WAIT_INTERRUPTIBLE,
|
|
MAX_SCHEDULE_TIMEOUT);
|
|
if (timeout < 0)
|
|
return timeout;
|
|
|
|
i915_request_retire_upto(target);
|
|
|
|
intel_ring_update_space(ring);
|
|
GEM_BUG_ON(ring->space < bytes);
|
|
return 0;
|
|
}
|
|
|
|
u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
|
|
{
|
|
struct intel_ring *ring = rq->ring;
|
|
const unsigned int remain_usable = ring->effective_size - ring->emit;
|
|
const unsigned int bytes = num_dwords * sizeof(u32);
|
|
unsigned int need_wrap = 0;
|
|
unsigned int total_bytes;
|
|
u32 *cs;
|
|
|
|
/* Packets must be qword aligned. */
|
|
GEM_BUG_ON(num_dwords & 1);
|
|
|
|
total_bytes = bytes + rq->reserved_space;
|
|
GEM_BUG_ON(total_bytes > ring->effective_size);
|
|
|
|
if (unlikely(total_bytes > remain_usable)) {
|
|
const int remain_actual = ring->size - ring->emit;
|
|
|
|
if (bytes > remain_usable) {
|
|
/*
|
|
* Not enough space for the basic request. So need to
|
|
* flush out the remainder and then wait for
|
|
* base + reserved.
|
|
*/
|
|
total_bytes += remain_actual;
|
|
need_wrap = remain_actual | 1;
|
|
} else {
|
|
/*
|
|
* The base request will fit but the reserved space
|
|
* falls off the end. So we don't need an immediate
|
|
* wrap and only need to effectively wait for the
|
|
* reserved size from the start of ringbuffer.
|
|
*/
|
|
total_bytes = rq->reserved_space + remain_actual;
|
|
}
|
|
}
|
|
|
|
if (unlikely(total_bytes > ring->space)) {
|
|
int ret;
|
|
|
|
/*
|
|
* Space is reserved in the ringbuffer for finalising the
|
|
* request, as that cannot be allowed to fail. During request
|
|
* finalisation, reserved_space is set to 0 to stop the
|
|
* overallocation and the assumption is that then we never need
|
|
* to wait (which has the risk of failing with EINTR).
|
|
*
|
|
* See also i915_request_alloc() and i915_request_add().
|
|
*/
|
|
GEM_BUG_ON(!rq->reserved_space);
|
|
|
|
ret = wait_for_space(ring,
|
|
i915_request_timeline(rq),
|
|
total_bytes);
|
|
if (unlikely(ret))
|
|
return ERR_PTR(ret);
|
|
}
|
|
|
|
if (unlikely(need_wrap)) {
|
|
need_wrap &= ~1;
|
|
GEM_BUG_ON(need_wrap > ring->space);
|
|
GEM_BUG_ON(ring->emit + need_wrap > ring->size);
|
|
GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64)));
|
|
|
|
/* Fill the tail with MI_NOOP */
|
|
memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64));
|
|
ring->space -= need_wrap;
|
|
ring->emit = 0;
|
|
}
|
|
|
|
GEM_BUG_ON(ring->emit > ring->size - bytes);
|
|
GEM_BUG_ON(ring->space < bytes);
|
|
cs = ring->vaddr + ring->emit;
|
|
GEM_DEBUG_EXEC(memset32(cs, POISON_INUSE, bytes / sizeof(*cs)));
|
|
ring->emit += bytes;
|
|
ring->space -= bytes;
|
|
|
|
return cs;
|
|
}
|
|
|
|
/* Align the ring tail to a cacheline boundary */
|
|
int intel_ring_cacheline_align(struct i915_request *rq)
|
|
{
|
|
int num_dwords;
|
|
void *cs;
|
|
|
|
num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
|
|
if (num_dwords == 0)
|
|
return 0;
|
|
|
|
num_dwords = CACHELINE_DWORDS - num_dwords;
|
|
GEM_BUG_ON(num_dwords & 1);
|
|
|
|
cs = intel_ring_begin(rq, num_dwords);
|
|
if (IS_ERR(cs))
|
|
return PTR_ERR(cs);
|
|
|
|
memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);
|
|
intel_ring_advance(rq, cs + num_dwords);
|
|
|
|
GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
|
|
return 0;
|
|
}
|