linux_dsm_epyc7002/drivers/gpu/drm/nouveau/nouveau_dmem.c

870 lines
21 KiB
C
Raw Normal View History

/*
* Copyright 2018 Red Hat Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "nouveau_dmem.h"
#include "nouveau_drv.h"
#include "nouveau_chan.h"
#include "nouveau_dma.h"
#include "nouveau_mem.h"
#include "nouveau_bo.h"
#include <nvif/class.h>
#include <nvif/object.h>
#include <nvif/if500b.h>
#include <nvif/if900b.h>
#include <linux/sched/mm.h>
#include <linux/hmm.h>
/*
* FIXME: this is ugly right now we are using TTM to allocate vram and we pin
* it in vram while in use. We likely want to overhaul memory management for
* nouveau to be more page like (not necessarily with system page size but a
* bigger page size) at lowest level and have some shim layer on top that would
* provide the same functionality as TTM.
*/
#define DMEM_CHUNK_SIZE (2UL << 20)
#define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
struct nouveau_migrate;
enum nouveau_aper {
NOUVEAU_APER_VIRT,
NOUVEAU_APER_VRAM,
NOUVEAU_APER_HOST,
};
typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
enum nouveau_aper, u64 dst_addr,
enum nouveau_aper, u64 src_addr);
struct nouveau_dmem_chunk {
struct list_head list;
struct nouveau_bo *bo;
struct nouveau_drm *drm;
unsigned long pfn_first;
unsigned long callocated;
unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
spinlock_t lock;
};
struct nouveau_dmem_migrate {
nouveau_migrate_copy_t copy_func;
struct nouveau_channel *chan;
};
struct nouveau_dmem {
struct nouveau_drm *drm;
struct dev_pagemap pagemap;
struct nouveau_dmem_migrate migrate;
struct list_head chunk_free;
struct list_head chunk_full;
struct list_head chunk_empty;
struct mutex mutex;
};
static inline struct nouveau_dmem *page_to_dmem(struct page *page)
{
return container_of(page->pgmap, struct nouveau_dmem, pagemap);
}
struct nouveau_dmem_fault {
struct nouveau_drm *drm;
struct nouveau_fence *fence;
dma_addr_t *dma;
unsigned long npages;
};
struct nouveau_migrate {
struct vm_area_struct *vma;
struct nouveau_drm *drm;
struct nouveau_fence *fence;
unsigned long npages;
dma_addr_t *dma;
unsigned long dma_nr;
};
static void nouveau_dmem_page_free(struct page *page)
{
struct nouveau_dmem_chunk *chunk = page->zone_device_data;
unsigned long idx = page_to_pfn(page) - chunk->pfn_first;
/*
* FIXME:
*
* This is really a bad example, we need to overhaul nouveau memory
* management to be more page focus and allow lighter locking scheme
* to be use in the process.
*/
spin_lock(&chunk->lock);
clear_bit(idx, chunk->bitmap);
WARN_ON(!chunk->callocated);
chunk->callocated--;
/*
* FIXME when chunk->callocated reach 0 we should add the chunk to
* a reclaim list so that it can be freed in case of memory pressure.
*/
spin_unlock(&chunk->lock);
}
static void
nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
const unsigned long *src_pfns,
unsigned long *dst_pfns,
unsigned long start,
unsigned long end,
void *private)
{
struct nouveau_dmem_fault *fault = private;
struct nouveau_drm *drm = fault->drm;
struct device *dev = drm->dev->dev;
unsigned long addr, i, npages = 0;
nouveau_migrate_copy_t copy;
int ret;
/* First allocate new memory */
for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
struct page *dpage, *spage;
dst_pfns[i] = 0;
spage = migrate_pfn_to_page(src_pfns[i]);
if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
continue;
dpage = alloc_page_vma(GFP_HIGHUSER, vma, addr);
if (!dpage) {
dst_pfns[i] = MIGRATE_PFN_ERROR;
continue;
}
lock_page(dpage);
dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
MIGRATE_PFN_LOCKED;
npages++;
}
/* Allocate storage for DMA addresses, so we can unmap later. */
fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL);
if (!fault->dma)
goto error;
/* Copy things over */
copy = drm->dmem->migrate.copy_func;
for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
struct nouveau_dmem_chunk *chunk;
struct page *spage, *dpage;
u64 src_addr, dst_addr;
dpage = migrate_pfn_to_page(dst_pfns[i]);
if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
continue;
spage = migrate_pfn_to_page(src_pfns[i]);
if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
dst_pfns[i] = MIGRATE_PFN_ERROR;
__free_page(dpage);
continue;
}
fault->dma[fault->npages] =
dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL,
DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(dev, fault->dma[fault->npages])) {
dst_pfns[i] = MIGRATE_PFN_ERROR;
__free_page(dpage);
continue;
}
dst_addr = fault->dma[fault->npages++];
chunk = spage->zone_device_data;
src_addr = page_to_pfn(spage) - chunk->pfn_first;
src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr,
NOUVEAU_APER_VRAM, src_addr);
if (ret) {
dst_pfns[i] = MIGRATE_PFN_ERROR;
__free_page(dpage);
continue;
}
}
nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
return;
error:
for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
struct page *page;
if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
continue;
page = migrate_pfn_to_page(dst_pfns[i]);
dst_pfns[i] = MIGRATE_PFN_ERROR;
if (page == NULL)
continue;
__free_page(page);
}
}
void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
const unsigned long *src_pfns,
const unsigned long *dst_pfns,
unsigned long start,
unsigned long end,
void *private)
{
struct nouveau_dmem_fault *fault = private;
struct nouveau_drm *drm = fault->drm;
if (fault->fence) {
nouveau_fence_wait(fault->fence, true, false);
nouveau_fence_unref(&fault->fence);
} else {
/*
* FIXME wait for channel to be IDLE before calling finalizing
* the hmem object below (nouveau_migrate_hmem_fini()).
*/
}
while (fault->npages--) {
dma_unmap_page(drm->dev->dev, fault->dma[fault->npages],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
}
kfree(fault->dma);
}
static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
.alloc_and_copy = nouveau_dmem_fault_alloc_and_copy,
.finalize_and_map = nouveau_dmem_fault_finalize_and_map,
};
static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
{
struct nouveau_dmem *dmem = page_to_dmem(vmf->page);
unsigned long src[1] = {0}, dst[1] = {0};
struct nouveau_dmem_fault fault = { .drm = dmem->drm };
int ret;
/*
* FIXME what we really want is to find some heuristic to migrate more
* than just one page on CPU fault. When such fault happens it is very
* likely that more surrounding page will CPU fault too.
*/
ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vmf->vma,
vmf->address, vmf->address + PAGE_SIZE,
src, dst, &fault);
if (ret)
return VM_FAULT_SIGBUS;
if (dst[0] == MIGRATE_PFN_ERROR)
return VM_FAULT_SIGBUS;
return 0;
}
static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = {
.page_free = nouveau_dmem_page_free,
.migrate_to_ram = nouveau_dmem_migrate_to_ram,
};
static int
nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
{
struct nouveau_dmem_chunk *chunk;
int ret;
if (drm->dmem == NULL)
return -EINVAL;
mutex_lock(&drm->dmem->mutex);
chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
struct nouveau_dmem_chunk,
list);
if (chunk == NULL) {
mutex_unlock(&drm->dmem->mutex);
return -ENOMEM;
}
list_del(&chunk->list);
mutex_unlock(&drm->dmem->mutex);
ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
&chunk->bo);
if (ret)
goto out;
ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
if (ret) {
nouveau_bo_ref(NULL, &chunk->bo);
goto out;
}
bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
spin_lock_init(&chunk->lock);
out:
mutex_lock(&drm->dmem->mutex);
if (chunk->bo)
list_add(&chunk->list, &drm->dmem->chunk_empty);
else
list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
mutex_unlock(&drm->dmem->mutex);
return ret;
}
static struct nouveau_dmem_chunk *
nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
{
struct nouveau_dmem_chunk *chunk;
chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
struct nouveau_dmem_chunk,
list);
if (chunk)
return chunk;
chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
struct nouveau_dmem_chunk,
list);
if (chunk->bo)
return chunk;
return NULL;
}
static int
nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
unsigned long npages,
unsigned long *pages)
{
struct nouveau_dmem_chunk *chunk;
unsigned long c;
int ret;
memset(pages, 0xff, npages * sizeof(*pages));
mutex_lock(&drm->dmem->mutex);
for (c = 0; c < npages;) {
unsigned long i;
chunk = nouveau_dmem_chunk_first_free_locked(drm);
if (chunk == NULL) {
mutex_unlock(&drm->dmem->mutex);
ret = nouveau_dmem_chunk_alloc(drm);
if (ret) {
if (c)
drm/nouveau/dmem: missing mutex_lock in error path In nouveau_dmem_pages_alloc(), the drm->dmem->mutex is unlocked before calling nouveau_dmem_chunk_alloc() as shown when CONFIG_PROVE_LOCKING is enabled: [ 1294.871933] ===================================== [ 1294.876656] WARNING: bad unlock balance detected! [ 1294.881375] 5.2.0-rc3+ #5 Not tainted [ 1294.885048] ------------------------------------- [ 1294.889773] test-malloc-vra/6299 is trying to release lock (&drm->dmem->mutex) at: [ 1294.897482] [<ffffffffa01a220f>] nouveau_dmem_migrate_alloc_and_copy+0x79f/0xbf0 [nouveau] [ 1294.905782] but there are no more locks to release! [ 1294.910690] [ 1294.910690] other info that might help us debug this: [ 1294.917249] 1 lock held by test-malloc-vra/6299: [ 1294.921881] #0: 0000000016e10454 (&mm->mmap_sem#2){++++}, at: nouveau_svmm_bind+0x142/0x210 [nouveau] [ 1294.931313] [ 1294.931313] stack backtrace: [ 1294.935702] CPU: 4 PID: 6299 Comm: test-malloc-vra Not tainted 5.2.0-rc3+ #5 [ 1294.942786] Hardware name: ASUS X299-A/PRIME X299-A, BIOS 1401 05/21/2018 [ 1294.949590] Call Trace: [ 1294.952059] dump_stack+0x7c/0xc0 [ 1294.955469] ? nouveau_dmem_migrate_alloc_and_copy+0x79f/0xbf0 [nouveau] [ 1294.962213] print_unlock_imbalance_bug.cold.52+0xca/0xcf [ 1294.967641] lock_release+0x306/0x380 [ 1294.971383] ? nouveau_dmem_migrate_alloc_and_copy+0x79f/0xbf0 [nouveau] [ 1294.978089] ? lock_downgrade+0x2d0/0x2d0 [ 1294.982121] ? find_held_lock+0xac/0xd0 [ 1294.985979] __mutex_unlock_slowpath+0x8f/0x3f0 [ 1294.990540] ? wait_for_completion+0x230/0x230 [ 1294.995002] ? rwlock_bug.part.2+0x60/0x60 [ 1294.999197] nouveau_dmem_migrate_alloc_and_copy+0x79f/0xbf0 [nouveau] [ 1295.005751] ? page_mapping+0x98/0x110 [ 1295.009511] migrate_vma+0xa74/0x1090 [ 1295.013186] ? move_to_new_page+0x480/0x480 [ 1295.017400] ? __kmalloc+0x153/0x300 [ 1295.021052] ? nouveau_dmem_migrate_vma+0xd8/0x1e0 [nouveau] [ 1295.026796] nouveau_dmem_migrate_vma+0x157/0x1e0 [nouveau] [ 1295.032466] ? nouveau_dmem_init+0x490/0x490 [nouveau] [ 1295.037612] ? vmacache_find+0xc2/0x110 [ 1295.041537] nouveau_svmm_bind+0x1b4/0x210 [nouveau] [ 1295.046583] ? nouveau_svm_fault+0x13e0/0x13e0 [nouveau] [ 1295.051912] drm_ioctl_kernel+0x14d/0x1a0 [ 1295.055930] ? drm_setversion+0x330/0x330 [ 1295.059971] drm_ioctl+0x308/0x530 [ 1295.063384] ? drm_version+0x150/0x150 [ 1295.067153] ? find_held_lock+0xac/0xd0 [ 1295.070996] ? __pm_runtime_resume+0x3f/0xa0 [ 1295.075285] ? mark_held_locks+0x29/0xa0 [ 1295.079230] ? _raw_spin_unlock_irqrestore+0x3c/0x50 [ 1295.084232] ? lockdep_hardirqs_on+0x17d/0x250 [ 1295.088768] nouveau_drm_ioctl+0x9a/0x100 [nouveau] [ 1295.093661] do_vfs_ioctl+0x137/0x9a0 [ 1295.097341] ? ioctl_preallocate+0x140/0x140 [ 1295.101623] ? match_held_lock+0x1b/0x230 [ 1295.105646] ? match_held_lock+0x1b/0x230 [ 1295.109660] ? find_held_lock+0xac/0xd0 [ 1295.113512] ? __do_page_fault+0x324/0x630 [ 1295.117617] ? lock_downgrade+0x2d0/0x2d0 [ 1295.121648] ? mark_held_locks+0x79/0xa0 [ 1295.125583] ? handle_mm_fault+0x352/0x430 [ 1295.129687] ksys_ioctl+0x60/0x90 [ 1295.133020] ? mark_held_locks+0x29/0xa0 [ 1295.136964] __x64_sys_ioctl+0x3d/0x50 [ 1295.140726] do_syscall_64+0x68/0x250 [ 1295.144400] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 1295.149465] RIP: 0033:0x7f1a3495809b [ 1295.153053] Code: 0f 1e fa 48 8b 05 ed bd 0c 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 0f 1f 44 00 00 f3 0f 1e fa b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d bd bd 0c 00 f7 d8 64 89 01 48 [ 1295.171850] RSP: 002b:00007ffef7ed1358 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 [ 1295.179451] RAX: ffffffffffffffda RBX: 00007ffef7ed1628 RCX: 00007f1a3495809b [ 1295.186601] RDX: 00007ffef7ed13b0 RSI: 0000000040406449 RDI: 0000000000000004 [ 1295.193759] RBP: 00007ffef7ed13b0 R08: 0000000000000000 R09: 000000000157e770 [ 1295.200917] R10: 000000000151c010 R11: 0000000000000246 R12: 0000000040406449 [ 1295.208083] R13: 0000000000000004 R14: 0000000000000000 R15: 0000000000000000 Reacquire the lock before continuing to the next page. Signed-off-by: Ralph Campbell <rcampbell@nvidia.com> Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
2019-06-15 03:20:03 +07:00
return 0;
return ret;
}
drm/nouveau/dmem: missing mutex_lock in error path In nouveau_dmem_pages_alloc(), the drm->dmem->mutex is unlocked before calling nouveau_dmem_chunk_alloc() as shown when CONFIG_PROVE_LOCKING is enabled: [ 1294.871933] ===================================== [ 1294.876656] WARNING: bad unlock balance detected! [ 1294.881375] 5.2.0-rc3+ #5 Not tainted [ 1294.885048] ------------------------------------- [ 1294.889773] test-malloc-vra/6299 is trying to release lock (&drm->dmem->mutex) at: [ 1294.897482] [<ffffffffa01a220f>] nouveau_dmem_migrate_alloc_and_copy+0x79f/0xbf0 [nouveau] [ 1294.905782] but there are no more locks to release! [ 1294.910690] [ 1294.910690] other info that might help us debug this: [ 1294.917249] 1 lock held by test-malloc-vra/6299: [ 1294.921881] #0: 0000000016e10454 (&mm->mmap_sem#2){++++}, at: nouveau_svmm_bind+0x142/0x210 [nouveau] [ 1294.931313] [ 1294.931313] stack backtrace: [ 1294.935702] CPU: 4 PID: 6299 Comm: test-malloc-vra Not tainted 5.2.0-rc3+ #5 [ 1294.942786] Hardware name: ASUS X299-A/PRIME X299-A, BIOS 1401 05/21/2018 [ 1294.949590] Call Trace: [ 1294.952059] dump_stack+0x7c/0xc0 [ 1294.955469] ? nouveau_dmem_migrate_alloc_and_copy+0x79f/0xbf0 [nouveau] [ 1294.962213] print_unlock_imbalance_bug.cold.52+0xca/0xcf [ 1294.967641] lock_release+0x306/0x380 [ 1294.971383] ? nouveau_dmem_migrate_alloc_and_copy+0x79f/0xbf0 [nouveau] [ 1294.978089] ? lock_downgrade+0x2d0/0x2d0 [ 1294.982121] ? find_held_lock+0xac/0xd0 [ 1294.985979] __mutex_unlock_slowpath+0x8f/0x3f0 [ 1294.990540] ? wait_for_completion+0x230/0x230 [ 1294.995002] ? rwlock_bug.part.2+0x60/0x60 [ 1294.999197] nouveau_dmem_migrate_alloc_and_copy+0x79f/0xbf0 [nouveau] [ 1295.005751] ? page_mapping+0x98/0x110 [ 1295.009511] migrate_vma+0xa74/0x1090 [ 1295.013186] ? move_to_new_page+0x480/0x480 [ 1295.017400] ? __kmalloc+0x153/0x300 [ 1295.021052] ? nouveau_dmem_migrate_vma+0xd8/0x1e0 [nouveau] [ 1295.026796] nouveau_dmem_migrate_vma+0x157/0x1e0 [nouveau] [ 1295.032466] ? nouveau_dmem_init+0x490/0x490 [nouveau] [ 1295.037612] ? vmacache_find+0xc2/0x110 [ 1295.041537] nouveau_svmm_bind+0x1b4/0x210 [nouveau] [ 1295.046583] ? nouveau_svm_fault+0x13e0/0x13e0 [nouveau] [ 1295.051912] drm_ioctl_kernel+0x14d/0x1a0 [ 1295.055930] ? drm_setversion+0x330/0x330 [ 1295.059971] drm_ioctl+0x308/0x530 [ 1295.063384] ? drm_version+0x150/0x150 [ 1295.067153] ? find_held_lock+0xac/0xd0 [ 1295.070996] ? __pm_runtime_resume+0x3f/0xa0 [ 1295.075285] ? mark_held_locks+0x29/0xa0 [ 1295.079230] ? _raw_spin_unlock_irqrestore+0x3c/0x50 [ 1295.084232] ? lockdep_hardirqs_on+0x17d/0x250 [ 1295.088768] nouveau_drm_ioctl+0x9a/0x100 [nouveau] [ 1295.093661] do_vfs_ioctl+0x137/0x9a0 [ 1295.097341] ? ioctl_preallocate+0x140/0x140 [ 1295.101623] ? match_held_lock+0x1b/0x230 [ 1295.105646] ? match_held_lock+0x1b/0x230 [ 1295.109660] ? find_held_lock+0xac/0xd0 [ 1295.113512] ? __do_page_fault+0x324/0x630 [ 1295.117617] ? lock_downgrade+0x2d0/0x2d0 [ 1295.121648] ? mark_held_locks+0x79/0xa0 [ 1295.125583] ? handle_mm_fault+0x352/0x430 [ 1295.129687] ksys_ioctl+0x60/0x90 [ 1295.133020] ? mark_held_locks+0x29/0xa0 [ 1295.136964] __x64_sys_ioctl+0x3d/0x50 [ 1295.140726] do_syscall_64+0x68/0x250 [ 1295.144400] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 1295.149465] RIP: 0033:0x7f1a3495809b [ 1295.153053] Code: 0f 1e fa 48 8b 05 ed bd 0c 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 0f 1f 44 00 00 f3 0f 1e fa b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d bd bd 0c 00 f7 d8 64 89 01 48 [ 1295.171850] RSP: 002b:00007ffef7ed1358 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 [ 1295.179451] RAX: ffffffffffffffda RBX: 00007ffef7ed1628 RCX: 00007f1a3495809b [ 1295.186601] RDX: 00007ffef7ed13b0 RSI: 0000000040406449 RDI: 0000000000000004 [ 1295.193759] RBP: 00007ffef7ed13b0 R08: 0000000000000000 R09: 000000000157e770 [ 1295.200917] R10: 000000000151c010 R11: 0000000000000246 R12: 0000000040406449 [ 1295.208083] R13: 0000000000000004 R14: 0000000000000000 R15: 0000000000000000 Reacquire the lock before continuing to the next page. Signed-off-by: Ralph Campbell <rcampbell@nvidia.com> Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
2019-06-15 03:20:03 +07:00
mutex_lock(&drm->dmem->mutex);
continue;
}
spin_lock(&chunk->lock);
i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
while (i < DMEM_CHUNK_NPAGES && c < npages) {
pages[c] = chunk->pfn_first + i;
set_bit(i, chunk->bitmap);
chunk->callocated++;
c++;
i = find_next_zero_bit(chunk->bitmap,
DMEM_CHUNK_NPAGES, i);
}
spin_unlock(&chunk->lock);
}
mutex_unlock(&drm->dmem->mutex);
return 0;
}
static struct page *
nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
{
unsigned long pfns[1];
struct page *page;
int ret;
/* FIXME stop all the miss-match API ... */
ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
if (ret)
return NULL;
page = pfn_to_page(pfns[0]);
get_page(page);
lock_page(page);
return page;
}
static void
nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
{
unlock_page(page);
put_page(page);
}
void
nouveau_dmem_resume(struct nouveau_drm *drm)
{
struct nouveau_dmem_chunk *chunk;
int ret;
if (drm->dmem == NULL)
return;
mutex_lock(&drm->dmem->mutex);
list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
/* FIXME handle pin failure */
WARN_ON(ret);
}
list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
/* FIXME handle pin failure */
WARN_ON(ret);
}
mutex_unlock(&drm->dmem->mutex);
}
void
nouveau_dmem_suspend(struct nouveau_drm *drm)
{
struct nouveau_dmem_chunk *chunk;
if (drm->dmem == NULL)
return;
mutex_lock(&drm->dmem->mutex);
list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
nouveau_bo_unpin(chunk->bo);
}
list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
nouveau_bo_unpin(chunk->bo);
}
mutex_unlock(&drm->dmem->mutex);
}
void
nouveau_dmem_fini(struct nouveau_drm *drm)
{
struct nouveau_dmem_chunk *chunk, *tmp;
if (drm->dmem == NULL)
return;
mutex_lock(&drm->dmem->mutex);
WARN_ON(!list_empty(&drm->dmem->chunk_free));
WARN_ON(!list_empty(&drm->dmem->chunk_full));
list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
if (chunk->bo) {
nouveau_bo_unpin(chunk->bo);
nouveau_bo_ref(NULL, &chunk->bo);
}
list_del(&chunk->list);
kfree(chunk);
}
mutex_unlock(&drm->dmem->mutex);
}
static int
nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
enum nouveau_aper dst_aper, u64 dst_addr,
enum nouveau_aper src_aper, u64 src_addr)
{
struct nouveau_channel *chan = drm->dmem->migrate.chan;
u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
(1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
(1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
(1 << 2) /* FLUSH_ENABLE_TRUE. */ |
(2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
int ret;
ret = RING_SPACE(chan, 13);
if (ret)
return ret;
if (src_aper != NOUVEAU_APER_VIRT) {
switch (src_aper) {
case NOUVEAU_APER_VRAM:
BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
break;
case NOUVEAU_APER_HOST:
BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
break;
default:
return -EINVAL;
}
launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
}
if (dst_aper != NOUVEAU_APER_VIRT) {
switch (dst_aper) {
case NOUVEAU_APER_VRAM:
BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
break;
case NOUVEAU_APER_HOST:
BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
break;
default:
return -EINVAL;
}
launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
}
BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
OUT_RING (chan, upper_32_bits(src_addr));
OUT_RING (chan, lower_32_bits(src_addr));
OUT_RING (chan, upper_32_bits(dst_addr));
OUT_RING (chan, lower_32_bits(dst_addr));
OUT_RING (chan, PAGE_SIZE);
OUT_RING (chan, PAGE_SIZE);
OUT_RING (chan, PAGE_SIZE);
OUT_RING (chan, npages);
BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
OUT_RING (chan, launch_dma);
return 0;
}
static int
nouveau_dmem_migrate_init(struct nouveau_drm *drm)
{
switch (drm->ttm.copy.oclass) {
case PASCAL_DMA_COPY_A:
case PASCAL_DMA_COPY_B:
case VOLTA_DMA_COPY_A:
case TURING_DMA_COPY_A:
drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
drm->dmem->migrate.chan = drm->ttm.chan;
return 0;
default:
break;
}
return -ENODEV;
}
void
nouveau_dmem_init(struct nouveau_drm *drm)
{
struct device *device = drm->dev->dev;
struct resource *res;
unsigned long i, size, pfn_first;
int ret;
/* This only make sense on PASCAL or newer */
if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
return;
if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
return;
drm->dmem->drm = drm;
mutex_init(&drm->dmem->mutex);
INIT_LIST_HEAD(&drm->dmem->chunk_free);
INIT_LIST_HEAD(&drm->dmem->chunk_full);
INIT_LIST_HEAD(&drm->dmem->chunk_empty);
size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
/* Initialize migration dma helpers before registering memory */
ret = nouveau_dmem_migrate_init(drm);
if (ret)
goto out_free;
/*
* FIXME we need some kind of policy to decide how much VRAM we
* want to register with HMM. For now just register everything
* and latter if we want to do thing like over commit then we
* could revisit this.
*/
res = devm_request_free_mem_region(device, &iomem_resource, size);
if (IS_ERR(res))
goto out_free;
drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
drm->dmem->pagemap.res = *res;
drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops;
if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap)))
goto out_free;
pfn_first = res->start >> PAGE_SHIFT;
for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
struct nouveau_dmem_chunk *chunk;
struct page *page;
unsigned long j;
chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
if (chunk == NULL) {
nouveau_dmem_fini(drm);
return;
}
chunk->drm = drm;
chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES);
list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
page = pfn_to_page(chunk->pfn_first);
for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page)
page->zone_device_data = chunk;
}
NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
return;
out_free:
kfree(drm->dmem);
drm->dmem = NULL;
}
static void
nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
const unsigned long *src_pfns,
unsigned long *dst_pfns,
unsigned long start,
unsigned long end,
void *private)
{
struct nouveau_migrate *migrate = private;
struct nouveau_drm *drm = migrate->drm;
struct device *dev = drm->dev->dev;
unsigned long addr, i, npages = 0;
nouveau_migrate_copy_t copy;
int ret;
/* First allocate new memory */
for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
struct page *dpage, *spage;
dst_pfns[i] = 0;
spage = migrate_pfn_to_page(src_pfns[i]);
if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
continue;
dpage = nouveau_dmem_page_alloc_locked(drm);
if (!dpage)
continue;
dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
MIGRATE_PFN_LOCKED |
MIGRATE_PFN_DEVICE;
npages++;
}
if (!npages)
return;
/* Allocate storage for DMA addresses, so we can unmap later. */
migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL);
if (!migrate->dma)
goto error;
/* Copy things over */
copy = drm->dmem->migrate.copy_func;
for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
struct nouveau_dmem_chunk *chunk;
struct page *spage, *dpage;
u64 src_addr, dst_addr;
dpage = migrate_pfn_to_page(dst_pfns[i]);
if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
continue;
chunk = dpage->zone_device_data;
dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
spage = migrate_pfn_to_page(src_pfns[i]);
if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
nouveau_dmem_page_free_locked(drm, dpage);
dst_pfns[i] = 0;
continue;
}
migrate->dma[migrate->dma_nr] =
dma_map_page_attrs(dev, spage, 0, PAGE_SIZE,
PCI_DMA_BIDIRECTIONAL,
DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) {
nouveau_dmem_page_free_locked(drm, dpage);
dst_pfns[i] = 0;
continue;
}
src_addr = migrate->dma[migrate->dma_nr++];
ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr,
NOUVEAU_APER_HOST, src_addr);
if (ret) {
nouveau_dmem_page_free_locked(drm, dpage);
dst_pfns[i] = 0;
continue;
}
}
nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
return;
error:
for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
struct page *page;
if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
continue;
page = migrate_pfn_to_page(dst_pfns[i]);
dst_pfns[i] = MIGRATE_PFN_ERROR;
if (page == NULL)
continue;
__free_page(page);
}
}
void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
const unsigned long *src_pfns,
const unsigned long *dst_pfns,
unsigned long start,
unsigned long end,
void *private)
{
struct nouveau_migrate *migrate = private;
struct nouveau_drm *drm = migrate->drm;
if (migrate->fence) {
nouveau_fence_wait(migrate->fence, true, false);
nouveau_fence_unref(&migrate->fence);
} else {
/*
* FIXME wait for channel to be IDLE before finalizing
* the hmem object below (nouveau_migrate_hmem_fini()) ?
*/
}
while (migrate->dma_nr--) {
dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr],
PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
}
kfree(migrate->dma);
/*
* FIXME optimization: update GPU page table to point to newly
* migrated memory.
*/
}
static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
.alloc_and_copy = nouveau_dmem_migrate_alloc_and_copy,
.finalize_and_map = nouveau_dmem_migrate_finalize_and_map,
};
int
nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
struct vm_area_struct *vma,
unsigned long start,
unsigned long end)
{
unsigned long *src_pfns, *dst_pfns, npages;
struct nouveau_migrate migrate = {0};
unsigned long i, c, max;
int ret = 0;
npages = (end - start) >> PAGE_SHIFT;
max = min(SG_MAX_SINGLE_ALLOC, npages);
src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
if (src_pfns == NULL)
return -ENOMEM;
dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
if (dst_pfns == NULL) {
kfree(src_pfns);
return -ENOMEM;
}
migrate.drm = drm;
migrate.vma = vma;
migrate.npages = npages;
for (i = 0; i < npages; i += c) {
unsigned long next;
c = min(SG_MAX_SINGLE_ALLOC, npages);
next = start + (c << PAGE_SHIFT);
ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
next, src_pfns, dst_pfns, &migrate);
if (ret)
goto out;
start = next;
}
out:
kfree(dst_pfns);
kfree(src_pfns);
return ret;
}
static inline bool
nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
{
return is_device_private_page(page) && drm->dmem == page_to_dmem(page);
}
void
nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
struct hmm_range *range)
{
unsigned long i, npages;
npages = (range->end - range->start) >> PAGE_SHIFT;
for (i = 0; i < npages; ++i) {
struct nouveau_dmem_chunk *chunk;
struct page *page;
uint64_t addr;
page = hmm_pfn_to_page(range, range->pfns[i]);
if (page == NULL)
continue;
if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
continue;
}
if (!nouveau_dmem_page(drm, page)) {
WARN(1, "Some unknown device memory !\n");
range->pfns[i] = 0;
continue;
}
chunk = page->zone_device_data;
addr = page_to_pfn(page) - chunk->pfn_first;
addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
}
}