mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-23 17:46:15 +07:00
91a8f1ea4b
The HW will only accept the DMA_FROM_MEMORY class for DMA_SEMAPHORE without asking the driver to intervene. It appears that semaphores will work correctly even without DMA_IN_MEMORY, so lets avoid the large amount of interrupts generated by x-chan sync. Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
618 lines
14 KiB
C
618 lines
14 KiB
C
/*
|
|
* Copyright (C) 2007 Ben Skeggs.
|
|
* All Rights Reserved.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining
|
|
* a copy of this software and associated documentation files (the
|
|
* "Software"), to deal in the Software without restriction, including
|
|
* without limitation the rights to use, copy, modify, merge, publish,
|
|
* distribute, sublicense, and/or sell copies of the Software, and to
|
|
* permit persons to whom the Software is furnished to do so, subject to
|
|
* the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice (including the
|
|
* next paragraph) shall be included in all copies or substantial
|
|
* portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
* IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
|
|
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
*
|
|
*/
|
|
|
|
#include "drmP.h"
|
|
#include "drm.h"
|
|
|
|
#include <linux/ktime.h>
|
|
#include <linux/hrtimer.h>
|
|
|
|
#include "nouveau_drv.h"
|
|
#include "nouveau_ramht.h"
|
|
#include "nouveau_dma.h"
|
|
|
|
#define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
|
|
#define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
|
|
|
|
struct nouveau_fence {
|
|
struct nouveau_channel *channel;
|
|
struct kref refcount;
|
|
struct list_head entry;
|
|
|
|
uint32_t sequence;
|
|
bool signalled;
|
|
|
|
void (*work)(void *priv, bool signalled);
|
|
void *priv;
|
|
};
|
|
|
|
struct nouveau_semaphore {
|
|
struct kref ref;
|
|
struct drm_device *dev;
|
|
struct drm_mm_node *mem;
|
|
};
|
|
|
|
static inline struct nouveau_fence *
|
|
nouveau_fence(void *sync_obj)
|
|
{
|
|
return (struct nouveau_fence *)sync_obj;
|
|
}
|
|
|
|
static void
|
|
nouveau_fence_del(struct kref *ref)
|
|
{
|
|
struct nouveau_fence *fence =
|
|
container_of(ref, struct nouveau_fence, refcount);
|
|
|
|
nouveau_channel_ref(NULL, &fence->channel);
|
|
kfree(fence);
|
|
}
|
|
|
|
void
|
|
nouveau_fence_update(struct nouveau_channel *chan)
|
|
{
|
|
struct drm_device *dev = chan->dev;
|
|
struct nouveau_fence *tmp, *fence;
|
|
uint32_t sequence;
|
|
|
|
spin_lock(&chan->fence.lock);
|
|
|
|
/* Fetch the last sequence if the channel is still up and running */
|
|
if (likely(!list_empty(&chan->fence.pending))) {
|
|
if (USE_REFCNT(dev))
|
|
sequence = nvchan_rd32(chan, 0x48);
|
|
else
|
|
sequence = atomic_read(&chan->fence.last_sequence_irq);
|
|
|
|
if (chan->fence.sequence_ack == sequence)
|
|
goto out;
|
|
chan->fence.sequence_ack = sequence;
|
|
}
|
|
|
|
list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
|
|
sequence = fence->sequence;
|
|
fence->signalled = true;
|
|
list_del(&fence->entry);
|
|
|
|
if (unlikely(fence->work))
|
|
fence->work(fence->priv, true);
|
|
|
|
kref_put(&fence->refcount, nouveau_fence_del);
|
|
|
|
if (sequence == chan->fence.sequence_ack)
|
|
break;
|
|
}
|
|
out:
|
|
spin_unlock(&chan->fence.lock);
|
|
}
|
|
|
|
int
|
|
nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
|
|
bool emit)
|
|
{
|
|
struct nouveau_fence *fence;
|
|
int ret = 0;
|
|
|
|
fence = kzalloc(sizeof(*fence), GFP_KERNEL);
|
|
if (!fence)
|
|
return -ENOMEM;
|
|
kref_init(&fence->refcount);
|
|
nouveau_channel_ref(chan, &fence->channel);
|
|
|
|
if (emit)
|
|
ret = nouveau_fence_emit(fence);
|
|
|
|
if (ret)
|
|
nouveau_fence_unref(&fence);
|
|
*pfence = fence;
|
|
return ret;
|
|
}
|
|
|
|
struct nouveau_channel *
|
|
nouveau_fence_channel(struct nouveau_fence *fence)
|
|
{
|
|
return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
|
|
}
|
|
|
|
int
|
|
nouveau_fence_emit(struct nouveau_fence *fence)
|
|
{
|
|
struct nouveau_channel *chan = fence->channel;
|
|
struct drm_device *dev = chan->dev;
|
|
struct drm_nouveau_private *dev_priv = dev->dev_private;
|
|
int ret;
|
|
|
|
ret = RING_SPACE(chan, 2);
|
|
if (ret)
|
|
return ret;
|
|
|
|
if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
|
|
nouveau_fence_update(chan);
|
|
|
|
BUG_ON(chan->fence.sequence ==
|
|
chan->fence.sequence_ack - 1);
|
|
}
|
|
|
|
fence->sequence = ++chan->fence.sequence;
|
|
|
|
kref_get(&fence->refcount);
|
|
spin_lock(&chan->fence.lock);
|
|
list_add_tail(&fence->entry, &chan->fence.pending);
|
|
spin_unlock(&chan->fence.lock);
|
|
|
|
if (USE_REFCNT(dev)) {
|
|
if (dev_priv->card_type < NV_C0)
|
|
BEGIN_RING(chan, NvSubSw, 0x0050, 1);
|
|
else
|
|
BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1);
|
|
} else {
|
|
BEGIN_RING(chan, NvSubSw, 0x0150, 1);
|
|
}
|
|
OUT_RING (chan, fence->sequence);
|
|
FIRE_RING(chan);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
nouveau_fence_work(struct nouveau_fence *fence,
|
|
void (*work)(void *priv, bool signalled),
|
|
void *priv)
|
|
{
|
|
BUG_ON(fence->work);
|
|
|
|
spin_lock(&fence->channel->fence.lock);
|
|
|
|
if (fence->signalled) {
|
|
work(priv, true);
|
|
} else {
|
|
fence->work = work;
|
|
fence->priv = priv;
|
|
}
|
|
|
|
spin_unlock(&fence->channel->fence.lock);
|
|
}
|
|
|
|
void
|
|
__nouveau_fence_unref(void **sync_obj)
|
|
{
|
|
struct nouveau_fence *fence = nouveau_fence(*sync_obj);
|
|
|
|
if (fence)
|
|
kref_put(&fence->refcount, nouveau_fence_del);
|
|
*sync_obj = NULL;
|
|
}
|
|
|
|
void *
|
|
__nouveau_fence_ref(void *sync_obj)
|
|
{
|
|
struct nouveau_fence *fence = nouveau_fence(sync_obj);
|
|
|
|
kref_get(&fence->refcount);
|
|
return sync_obj;
|
|
}
|
|
|
|
bool
|
|
__nouveau_fence_signalled(void *sync_obj, void *sync_arg)
|
|
{
|
|
struct nouveau_fence *fence = nouveau_fence(sync_obj);
|
|
struct nouveau_channel *chan = fence->channel;
|
|
|
|
if (fence->signalled)
|
|
return true;
|
|
|
|
nouveau_fence_update(chan);
|
|
return fence->signalled;
|
|
}
|
|
|
|
int
|
|
__nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
|
|
{
|
|
unsigned long timeout = jiffies + (3 * DRM_HZ);
|
|
unsigned long sleep_time = NSEC_PER_MSEC / 1000;
|
|
ktime_t t;
|
|
int ret = 0;
|
|
|
|
while (1) {
|
|
if (__nouveau_fence_signalled(sync_obj, sync_arg))
|
|
break;
|
|
|
|
if (time_after_eq(jiffies, timeout)) {
|
|
ret = -EBUSY;
|
|
break;
|
|
}
|
|
|
|
__set_current_state(intr ? TASK_INTERRUPTIBLE
|
|
: TASK_UNINTERRUPTIBLE);
|
|
if (lazy) {
|
|
t = ktime_set(0, sleep_time);
|
|
schedule_hrtimeout(&t, HRTIMER_MODE_REL);
|
|
sleep_time *= 2;
|
|
if (sleep_time > NSEC_PER_MSEC)
|
|
sleep_time = NSEC_PER_MSEC;
|
|
}
|
|
|
|
if (intr && signal_pending(current)) {
|
|
ret = -ERESTARTSYS;
|
|
break;
|
|
}
|
|
}
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static struct nouveau_semaphore *
|
|
semaphore_alloc(struct drm_device *dev)
|
|
{
|
|
struct drm_nouveau_private *dev_priv = dev->dev_private;
|
|
struct nouveau_semaphore *sema;
|
|
int size = (dev_priv->chipset < 0x84) ? 4 : 16;
|
|
int ret, i;
|
|
|
|
if (!USE_SEMA(dev))
|
|
return NULL;
|
|
|
|
sema = kmalloc(sizeof(*sema), GFP_KERNEL);
|
|
if (!sema)
|
|
goto fail;
|
|
|
|
ret = drm_mm_pre_get(&dev_priv->fence.heap);
|
|
if (ret)
|
|
goto fail;
|
|
|
|
spin_lock(&dev_priv->fence.lock);
|
|
sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
|
|
if (sema->mem)
|
|
sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
|
|
spin_unlock(&dev_priv->fence.lock);
|
|
|
|
if (!sema->mem)
|
|
goto fail;
|
|
|
|
kref_init(&sema->ref);
|
|
sema->dev = dev;
|
|
for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
|
|
nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
|
|
|
|
return sema;
|
|
fail:
|
|
kfree(sema);
|
|
return NULL;
|
|
}
|
|
|
|
static void
|
|
semaphore_free(struct kref *ref)
|
|
{
|
|
struct nouveau_semaphore *sema =
|
|
container_of(ref, struct nouveau_semaphore, ref);
|
|
struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
|
|
|
|
spin_lock(&dev_priv->fence.lock);
|
|
drm_mm_put_block(sema->mem);
|
|
spin_unlock(&dev_priv->fence.lock);
|
|
|
|
kfree(sema);
|
|
}
|
|
|
|
static void
|
|
semaphore_work(void *priv, bool signalled)
|
|
{
|
|
struct nouveau_semaphore *sema = priv;
|
|
struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
|
|
|
|
if (unlikely(!signalled))
|
|
nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
|
|
|
|
kref_put(&sema->ref, semaphore_free);
|
|
}
|
|
|
|
static int
|
|
semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
|
|
{
|
|
struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
|
|
struct nouveau_fence *fence = NULL;
|
|
u64 offset = chan->fence.vma.offset + sema->mem->start;
|
|
int ret;
|
|
|
|
if (dev_priv->chipset < 0x84) {
|
|
ret = RING_SPACE(chan, 4);
|
|
if (ret)
|
|
return ret;
|
|
|
|
BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3);
|
|
OUT_RING (chan, NvSema);
|
|
OUT_RING (chan, offset);
|
|
OUT_RING (chan, 1);
|
|
} else
|
|
if (dev_priv->chipset < 0xc0) {
|
|
ret = RING_SPACE(chan, 7);
|
|
if (ret)
|
|
return ret;
|
|
|
|
BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
|
|
OUT_RING (chan, chan->vram_handle);
|
|
BEGIN_RING(chan, NvSubSw, 0x0010, 4);
|
|
OUT_RING (chan, upper_32_bits(offset));
|
|
OUT_RING (chan, lower_32_bits(offset));
|
|
OUT_RING (chan, 1);
|
|
OUT_RING (chan, 1); /* ACQUIRE_EQ */
|
|
} else {
|
|
ret = RING_SPACE(chan, 5);
|
|
if (ret)
|
|
return ret;
|
|
|
|
BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
|
|
OUT_RING (chan, upper_32_bits(offset));
|
|
OUT_RING (chan, lower_32_bits(offset));
|
|
OUT_RING (chan, 1);
|
|
OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */
|
|
}
|
|
|
|
/* Delay semaphore destruction until its work is done */
|
|
ret = nouveau_fence_new(chan, &fence, true);
|
|
if (ret)
|
|
return ret;
|
|
|
|
kref_get(&sema->ref);
|
|
nouveau_fence_work(fence, semaphore_work, sema);
|
|
nouveau_fence_unref(&fence);
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
|
|
{
|
|
struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
|
|
struct nouveau_fence *fence = NULL;
|
|
u64 offset = chan->fence.vma.offset + sema->mem->start;
|
|
int ret;
|
|
|
|
if (dev_priv->chipset < 0x84) {
|
|
ret = RING_SPACE(chan, 5);
|
|
if (ret)
|
|
return ret;
|
|
|
|
BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2);
|
|
OUT_RING (chan, NvSema);
|
|
OUT_RING (chan, offset);
|
|
BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
|
|
OUT_RING (chan, 1);
|
|
} else
|
|
if (dev_priv->chipset < 0xc0) {
|
|
ret = RING_SPACE(chan, 7);
|
|
if (ret)
|
|
return ret;
|
|
|
|
BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
|
|
OUT_RING (chan, chan->vram_handle);
|
|
BEGIN_RING(chan, NvSubSw, 0x0010, 4);
|
|
OUT_RING (chan, upper_32_bits(offset));
|
|
OUT_RING (chan, lower_32_bits(offset));
|
|
OUT_RING (chan, 1);
|
|
OUT_RING (chan, 2); /* RELEASE */
|
|
} else {
|
|
ret = RING_SPACE(chan, 5);
|
|
if (ret)
|
|
return ret;
|
|
|
|
BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
|
|
OUT_RING (chan, upper_32_bits(offset));
|
|
OUT_RING (chan, lower_32_bits(offset));
|
|
OUT_RING (chan, 1);
|
|
OUT_RING (chan, 0x1002); /* RELEASE */
|
|
}
|
|
|
|
/* Delay semaphore destruction until its work is done */
|
|
ret = nouveau_fence_new(chan, &fence, true);
|
|
if (ret)
|
|
return ret;
|
|
|
|
kref_get(&sema->ref);
|
|
nouveau_fence_work(fence, semaphore_work, sema);
|
|
nouveau_fence_unref(&fence);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
nouveau_fence_sync(struct nouveau_fence *fence,
|
|
struct nouveau_channel *wchan)
|
|
{
|
|
struct nouveau_channel *chan = nouveau_fence_channel(fence);
|
|
struct drm_device *dev = wchan->dev;
|
|
struct nouveau_semaphore *sema;
|
|
int ret = 0;
|
|
|
|
if (likely(!chan || chan == wchan ||
|
|
nouveau_fence_signalled(fence)))
|
|
goto out;
|
|
|
|
sema = semaphore_alloc(dev);
|
|
if (!sema) {
|
|
/* Early card or broken userspace, fall back to
|
|
* software sync. */
|
|
ret = nouveau_fence_wait(fence, true, false);
|
|
goto out;
|
|
}
|
|
|
|
/* try to take chan's mutex, if we can't take it right away
|
|
* we have to fallback to software sync to prevent locking
|
|
* order issues
|
|
*/
|
|
if (!mutex_trylock(&chan->mutex)) {
|
|
ret = nouveau_fence_wait(fence, true, false);
|
|
goto out_unref;
|
|
}
|
|
|
|
/* Make wchan wait until it gets signalled */
|
|
ret = semaphore_acquire(wchan, sema);
|
|
if (ret)
|
|
goto out_unlock;
|
|
|
|
/* Signal the semaphore from chan */
|
|
ret = semaphore_release(chan, sema);
|
|
|
|
out_unlock:
|
|
mutex_unlock(&chan->mutex);
|
|
out_unref:
|
|
kref_put(&sema->ref, semaphore_free);
|
|
out:
|
|
if (chan)
|
|
nouveau_channel_put_unlocked(&chan);
|
|
return ret;
|
|
}
|
|
|
|
int
|
|
__nouveau_fence_flush(void *sync_obj, void *sync_arg)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
nouveau_fence_channel_init(struct nouveau_channel *chan)
|
|
{
|
|
struct drm_device *dev = chan->dev;
|
|
struct drm_nouveau_private *dev_priv = dev->dev_private;
|
|
struct nouveau_gpuobj *obj = NULL;
|
|
int ret;
|
|
|
|
if (dev_priv->card_type < NV_C0) {
|
|
/* Create an NV_SW object for various sync purposes */
|
|
ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = RING_SPACE(chan, 2);
|
|
if (ret)
|
|
return ret;
|
|
|
|
BEGIN_RING(chan, NvSubSw, 0, 1);
|
|
OUT_RING (chan, NvSw);
|
|
FIRE_RING (chan);
|
|
}
|
|
|
|
/* Setup area of memory shared between all channels for x-chan sync */
|
|
if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
|
|
struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
|
|
|
|
ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_FROM_MEMORY,
|
|
mem->start << PAGE_SHIFT,
|
|
mem->size, NV_MEM_ACCESS_RW,
|
|
NV_MEM_TARGET_VRAM, &obj);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = nouveau_ramht_insert(chan, NvSema, obj);
|
|
nouveau_gpuobj_ref(NULL, &obj);
|
|
if (ret)
|
|
return ret;
|
|
} else
|
|
if (USE_SEMA(dev)) {
|
|
/* map fence bo into channel's vm */
|
|
ret = nouveau_bo_vma_add(dev_priv->fence.bo, chan->vm,
|
|
&chan->fence.vma);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
INIT_LIST_HEAD(&chan->fence.pending);
|
|
spin_lock_init(&chan->fence.lock);
|
|
atomic_set(&chan->fence.last_sequence_irq, 0);
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
nouveau_fence_channel_fini(struct nouveau_channel *chan)
|
|
{
|
|
struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
|
|
struct nouveau_fence *tmp, *fence;
|
|
|
|
spin_lock(&chan->fence.lock);
|
|
list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
|
|
fence->signalled = true;
|
|
list_del(&fence->entry);
|
|
|
|
if (unlikely(fence->work))
|
|
fence->work(fence->priv, false);
|
|
|
|
kref_put(&fence->refcount, nouveau_fence_del);
|
|
}
|
|
spin_unlock(&chan->fence.lock);
|
|
|
|
nouveau_bo_vma_del(dev_priv->fence.bo, &chan->fence.vma);
|
|
}
|
|
|
|
int
|
|
nouveau_fence_init(struct drm_device *dev)
|
|
{
|
|
struct drm_nouveau_private *dev_priv = dev->dev_private;
|
|
int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
|
|
int ret;
|
|
|
|
/* Create a shared VRAM heap for cross-channel sync. */
|
|
if (USE_SEMA(dev)) {
|
|
ret = nouveau_bo_new(dev, size, 0, TTM_PL_FLAG_VRAM,
|
|
0, 0, &dev_priv->fence.bo);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
|
|
if (ret)
|
|
goto fail;
|
|
|
|
ret = nouveau_bo_map(dev_priv->fence.bo);
|
|
if (ret)
|
|
goto fail;
|
|
|
|
ret = drm_mm_init(&dev_priv->fence.heap, 0,
|
|
dev_priv->fence.bo->bo.mem.size);
|
|
if (ret)
|
|
goto fail;
|
|
|
|
spin_lock_init(&dev_priv->fence.lock);
|
|
}
|
|
|
|
return 0;
|
|
fail:
|
|
nouveau_bo_unmap(dev_priv->fence.bo);
|
|
nouveau_bo_ref(NULL, &dev_priv->fence.bo);
|
|
return ret;
|
|
}
|
|
|
|
void
|
|
nouveau_fence_fini(struct drm_device *dev)
|
|
{
|
|
struct drm_nouveau_private *dev_priv = dev->dev_private;
|
|
|
|
if (USE_SEMA(dev)) {
|
|
drm_mm_takedown(&dev_priv->fence.heap);
|
|
nouveau_bo_unmap(dev_priv->fence.bo);
|
|
nouveau_bo_unpin(dev_priv->fence.bo);
|
|
nouveau_bo_ref(NULL, &dev_priv->fence.bo);
|
|
}
|
|
}
|