AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-12-20 18:41:57 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

drm/i915/gtt: Use a common type for page directories

Browse Source 
All page directories are identical in function, only the position in the
hierarchy differ. Use same base type for directory functionality.

v2: cleanup, size always 512, init to null

Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Matthew Auld <matthew.william.auld@gmail.com>
Cc: Abdiel Janulgue <abdiel.janulgue@linux.intel.com>
Signed-off-by: Mika Kuoppala <mika.kuoppala@linux.intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20190614164350.30415-2-mika.kuoppala@linux.intel.com
This commit is contained in:
Mika Kuoppala 2019-06-14 19:43:42 +03:00
parent 7d82cc353a
commit b5b7bef9ca
6 changed files with 236 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/gpu/drm/i915/gem/i915_gem_context.c
View File

@ -1027,7 +1027,7 @@ static int emit_ppgtt_update(struct i915_request *rq, void *data)
if (i915_vm_is_4lvl(vm)) { if (i915_vm_is_4lvl(vm)) {
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
const dma_addr_t pd_daddr = px_dma(&ppgtt->pml4); const dma_addr_t pd_daddr = px_dma(ppgtt->pd);
cs = intel_ring_begin(rq, 6); cs = intel_ring_begin(rq, 6);
if (IS_ERR(cs)) if (IS_ERR(cs))

2
drivers/gpu/drm/i915/gt/intel_lrc_reg.h
View File

@ -55,7 +55,7 @@
#define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \ #define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \
u32 *reg_state__ = (reg_state); \ u32 *reg_state__ = (reg_state); \
const u64 addr__ = px_dma(&ppgtt->pml4); \ const u64 addr__ = px_dma(ppgtt->pd); \
(reg_state__)[CTX_PDP0_UDW + 1] = upper_32_bits(addr__); \ (reg_state__)[CTX_PDP0_UDW + 1] = upper_32_bits(addr__); \
(reg_state__)[CTX_PDP0_LDW + 1] = lower_32_bits(addr__); \ (reg_state__)[CTX_PDP0_LDW + 1] = lower_32_bits(addr__); \
} while (0) } while (0)

2
drivers/gpu/drm/i915/gt/intel_ringbuffer.c
View File

@ -1484,7 +1484,7 @@ static int load_pd_dir(struct i915_request *rq, const struct i915_ppgtt *ppgtt)
*cs++ = MI_LOAD_REGISTER_IMM(1); *cs++ = MI_LOAD_REGISTER_IMM(1);
*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
*cs++ = ppgtt->pd.base.ggtt_offset << 10; *cs++ = ppgtt->pd->base.ggtt_offset << 10;
intel_ring_advance(rq, cs); intel_ring_advance(rq, cs);

30
drivers/gpu/drm/i915/gvt/scheduler.c
View File

@ -375,11 +375,13 @@ static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload,
return -EINVAL; return -EINVAL;
if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
px_dma(&ppgtt->pml4) = mm->ppgtt_mm.shadow_pdps[0]; px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0];
} else { } else {
for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) { for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) {
px_dma(ppgtt->pdp.page_directory[i]) = struct i915_page_directory * const pd =
mm->ppgtt_mm.shadow_pdps[i]; i915_pd_entry(ppgtt->pd, i);
px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i];
} }
} }
@ -1107,11 +1109,14 @@ i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s,
int i; int i;
if (i915_vm_is_4lvl(&ppgtt->vm)) { if (i915_vm_is_4lvl(&ppgtt->vm)) {
px_dma(&ppgtt->pml4) = s->i915_context_pml4; px_dma(ppgtt->pd) = s->i915_context_pml4;
} else { } else {
for (i = 0; i < GEN8_3LVL_PDPES; i++) for (i = 0; i < GEN8_3LVL_PDPES; i++) {
px_dma(ppgtt->pdp.page_directory[i]) = struct i915_page_directory * const pd =
s->i915_context_pdps[i]; i915_pd_entry(ppgtt->pd, i);
px_dma(pd) = s->i915_context_pdps[i];
}
} }
} }
@ -1165,11 +1170,14 @@ i915_context_ppgtt_root_save(struct intel_vgpu_submission *s,
int i; int i;
if (i915_vm_is_4lvl(&ppgtt->vm)) { if (i915_vm_is_4lvl(&ppgtt->vm)) {
s->i915_context_pml4 = px_dma(&ppgtt->pml4); s->i915_context_pml4 = px_dma(ppgtt->pd);
} else { } else {
for (i = 0; i < GEN8_3LVL_PDPES; i++) for (i = 0; i < GEN8_3LVL_PDPES; i++) {
s->i915_context_pdps[i] = struct i915_page_directory * const pd =
px_dma(ppgtt->pdp.page_directory[i]); i915_pd_entry(ppgtt->pd, i);
s->i915_context_pdps[i] = px_dma(pd);
}
} }
} }

351
drivers/gpu/drm/i915/i915_gem_gtt.c
View File

@ -662,7 +662,8 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
} }
atomic_set(&pt->used_ptes, 0); atomic_set(&pt->used, 0);
return pt; return pt;
} }
@ -684,11 +685,30 @@ static void gen6_initialize_pt(struct i915_address_space *vm,
fill32_px(vm, pt, vm->scratch_pte); fill32_px(vm, pt, vm->scratch_pte);
} }
static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) static struct i915_page_directory *__alloc_pd(void)
{ {
struct i915_page_directory *pd; struct i915_page_directory *pd;
pd = kmalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL); pd = kmalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL);
if (unlikely(!pd))
return NULL;
memset(&pd->base, 0, sizeof(pd->base));
atomic_set(&pd->used, 0);
spin_lock_init(&pd->lock);
/* for safety */
pd->entry[0] = NULL;
return pd;
}
static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
{
struct i915_page_directory *pd;
pd = __alloc_pd();
if (unlikely(!pd)) if (unlikely(!pd))
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
@ -697,8 +717,6 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
} }
atomic_set(&pd->used_pdes, 0);
spin_lock_init(&pd->lock);
return pd; return pd;
} }
@ -714,88 +732,56 @@ static void gen8_initialize_pd(struct i915_address_space *vm,
{ {
fill_px(vm, pd, fill_px(vm, pd,
gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC)); gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));
memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES); memset_p(pd->entry, vm->scratch_pt, I915_PDES);
} }
static int __pdp_init(struct i915_address_space *vm, static struct i915_page_directory *alloc_pdp(struct i915_address_space *vm)
struct i915_page_directory_pointer *pdp)
{ {
const unsigned int pdpes = i915_pdpes_per_pdp(vm); struct i915_page_directory *pdp;
pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory), pdp = __alloc_pd();
I915_GFP_ALLOW_FAIL);
if (unlikely(!pdp->page_directory))
return -ENOMEM;
memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);
atomic_set(&pdp->used_pdpes, 0);
spin_lock_init(&pdp->lock);
return 0;
}
static void __pdp_fini(struct i915_page_directory_pointer *pdp)
{
kfree(pdp->page_directory);
pdp->page_directory = NULL;
}
static struct i915_page_directory_pointer *
alloc_pdp(struct i915_address_space *vm)
{
struct i915_page_directory_pointer *pdp;
int ret = -ENOMEM;
GEM_BUG_ON(!i915_vm_is_4lvl(vm));
pdp = kmalloc(sizeof(*pdp), I915_GFP_ALLOW_FAIL);
if (!pdp) if (!pdp)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
ret = __pdp_init(vm, pdp); if (i915_vm_is_4lvl(vm)) {
if (ret) if (unlikely(setup_px(vm, pdp))) {
goto fail_bitmap; kfree(pdp);
return ERR_PTR(-ENOMEM);
ret = setup_px(vm, pdp); }
if (ret) }
goto fail_page_m;
return pdp; return pdp;
fail_page_m:
__pdp_fini(pdp);
fail_bitmap:
kfree(pdp);
return ERR_PTR(ret);
} }
static void free_pdp(struct i915_address_space *vm, static void free_pdp(struct i915_address_space *vm,
struct i915_page_directory_pointer *pdp) struct i915_page_directory *pdp)
{ {
__pdp_fini(pdp); if (i915_vm_is_4lvl(vm))
cleanup_px(vm, pdp);
if (!i915_vm_is_4lvl(vm))
return;
cleanup_px(vm, pdp);
kfree(pdp); kfree(pdp);
} }
static void gen8_initialize_pdp(struct i915_address_space *vm, static void gen8_initialize_4lvl_pdp(struct i915_address_space *vm,
struct i915_page_directory_pointer *pdp) struct i915_page_directory *pdp)
{ {
fill_px(vm, pdp, fill_px(vm, pdp,
gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC)); gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC));
memset_p(pdp->entry, vm->scratch_pd, 512);
}
static void gen8_initialize_3lvl_pdp(struct i915_address_space *vm,
struct i915_page_directory *pdp)
{
memset_p(pdp->entry, vm->scratch_pd, GEN8_3LVL_PDPES);
} }
static void gen8_initialize_pml4(struct i915_address_space *vm, static void gen8_initialize_pml4(struct i915_address_space *vm,
struct i915_pml4 *pml4) struct i915_page_directory *pml4)
{ {
fill_px(vm, pml4, fill_px(vm, pml4,
gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC)); gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4); memset_p(pml4->entry, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
spin_lock_init(&pml4->lock);
} }
/* /*
@ -823,8 +809,8 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries); memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries);
kunmap_atomic(vaddr); kunmap_atomic(vaddr);
GEM_BUG_ON(num_entries > atomic_read(&pt->used_ptes)); GEM_BUG_ON(num_entries > atomic_read(&pt->used));
return !atomic_sub_return(num_entries, &pt->used_ptes); return !atomic_sub_return(num_entries, &pt->used);
} }
static void gen8_ppgtt_set_pde(struct i915_address_space *vm, static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
@ -855,12 +841,12 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
continue; continue;
spin_lock(&pd->lock); spin_lock(&pd->lock);
if (!atomic_read(&pt->used_ptes)) { if (!atomic_read(&pt->used)) {
gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde); gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
pd->page_table[pde] = vm->scratch_pt; pd->entry[pde] = vm->scratch_pt;
GEM_BUG_ON(!atomic_read(&pd->used_pdes)); GEM_BUG_ON(!atomic_read(&pd->used));
atomic_dec(&pd->used_pdes); atomic_dec(&pd->used);
free = true; free = true;
} }
spin_unlock(&pd->lock); spin_unlock(&pd->lock);
@ -868,11 +854,11 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
free_pt(vm, pt); free_pt(vm, pt);
} }
return !atomic_read(&pd->used_pdes); return !atomic_read(&pd->used);
} }
static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm, static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
struct i915_page_directory_pointer *pdp, struct i915_page_directory *pdp,
struct i915_page_directory *pd, struct i915_page_directory *pd,
unsigned int pdpe) unsigned int pdpe)
{ {
@ -890,7 +876,7 @@ static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
* Caller can use the return value to update higher-level entries * Caller can use the return value to update higher-level entries
*/ */
static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
struct i915_page_directory_pointer *pdp, struct i915_page_directory * const pdp,
u64 start, u64 length) u64 start, u64 length)
{ {
struct i915_page_directory *pd; struct i915_page_directory *pd;
@ -905,12 +891,12 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
continue; continue;
spin_lock(&pdp->lock); spin_lock(&pdp->lock);
if (!atomic_read(&pd->used_pdes)) { if (!atomic_read(&pd->used)) {
gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
pdp->page_directory[pdpe] = vm->scratch_pd; pdp->entry[pdpe] = vm->scratch_pd;
GEM_BUG_ON(!atomic_read(&pdp->used_pdpes)); GEM_BUG_ON(!atomic_read(&pdp->used));
atomic_dec(&pdp->used_pdpes); atomic_dec(&pdp->used);
free = true; free = true;
} }
spin_unlock(&pdp->lock); spin_unlock(&pdp->lock);
@ -918,17 +904,17 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
free_pd(vm, pd); free_pd(vm, pd);
} }
return !atomic_read(&pdp->used_pdpes); return !atomic_read(&pdp->used);
} }
static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm, static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
u64 start, u64 length) u64 start, u64 length)
{ {
gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length); gen8_ppgtt_clear_pdp(vm, i915_vm_to_ppgtt(vm)->pd, start, length);
} }
static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4, static void gen8_ppgtt_set_pml4e(struct i915_page_directory *pml4,
struct i915_page_directory_pointer *pdp, struct i915_page_directory *pdp,
unsigned int pml4e) unsigned int pml4e)
{ {
gen8_ppgtt_pml4e_t *vaddr; gen8_ppgtt_pml4e_t *vaddr;
@ -946,8 +932,8 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
u64 start, u64 length) u64 start, u64 length)
{ {
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
struct i915_pml4 *pml4 = &ppgtt->pml4; struct i915_page_directory * const pml4 = ppgtt->pd;
struct i915_page_directory_pointer *pdp; struct i915_page_directory *pdp;
unsigned int pml4e; unsigned int pml4e;
GEM_BUG_ON(!i915_vm_is_4lvl(vm)); GEM_BUG_ON(!i915_vm_is_4lvl(vm));
@ -960,9 +946,9 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
continue; continue;
spin_lock(&pml4->lock); spin_lock(&pml4->lock);
if (!atomic_read(&pdp->used_pdpes)) { if (!atomic_read(&pdp->used)) {
gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
pml4->pdps[pml4e] = vm->scratch_pdp; pml4->entry[pml4e] = vm->scratch_pdp;
free = true; free = true;
} }
spin_unlock(&pml4->lock); spin_unlock(&pml4->lock);
@ -999,7 +985,7 @@ static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
static __always_inline bool static __always_inline bool
gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt, gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
struct i915_page_directory_pointer *pdp, struct i915_page_directory *pdp,
struct sgt_dma *iter, struct sgt_dma *iter,
struct gen8_insert_pte *idx, struct gen8_insert_pte *idx,
enum i915_cache_level cache_level, enum i915_cache_level cache_level,
@ -1011,8 +997,8 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
bool ret; bool ret;
GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm)); GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
pd = pdp->page_directory[idx->pdpe]; pd = i915_pd_entry(pdp, idx->pdpe);
vaddr = kmap_atomic_px(pd->page_table[idx->pde]); vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
do { do {
vaddr[idx->pte] = pte_encode | iter->dma; vaddr[idx->pte] = pte_encode | iter->dma;
@ -1042,11 +1028,11 @@ gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
} }
GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm)); GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
pd = pdp->page_directory[idx->pdpe]; pd = pdp->entry[idx->pdpe];
} }
kunmap_atomic(vaddr); kunmap_atomic(vaddr);
vaddr = kmap_atomic_px(pd->page_table[idx->pde]); vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
} }
} while (1); } while (1);
kunmap_atomic(vaddr); kunmap_atomic(vaddr);
@ -1063,14 +1049,14 @@ static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
struct sgt_dma iter = sgt_dma(vma); struct sgt_dma iter = sgt_dma(vma);
struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start); struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx, gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter, &idx,
cache_level, flags); cache_level, flags);
vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
} }
static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
struct i915_page_directory_pointer **pdps, struct i915_page_directory *pml4,
struct sgt_dma *iter, struct sgt_dma *iter,
enum i915_cache_level cache_level, enum i915_cache_level cache_level,
u32 flags) u32 flags)
@ -1081,8 +1067,9 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
do { do {
struct gen8_insert_pte idx = gen8_insert_pte(start); struct gen8_insert_pte idx = gen8_insert_pte(start);
struct i915_page_directory_pointer *pdp = pdps[idx.pml4e]; struct i915_page_directory *pdp =
struct i915_page_directory *pd = pdp->page_directory[idx.pdpe]; i915_pdp_entry(pml4, idx.pml4e);
struct i915_page_directory *pd = i915_pd_entry(pdp, idx.pdpe);
unsigned int page_size; unsigned int page_size;
bool maybe_64K = false; bool maybe_64K = false;
gen8_pte_t encode = pte_encode; gen8_pte_t encode = pte_encode;
@ -1100,7 +1087,7 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
vaddr = kmap_atomic_px(pd); vaddr = kmap_atomic_px(pd);
} else { } else {
struct i915_page_table *pt = pd->page_table[idx.pde]; struct i915_page_table *pt = i915_pt_entry(pd, idx.pde);
index = idx.pte; index = idx.pte;
max = GEN8_PTES; max = GEN8_PTES;
@ -1175,7 +1162,8 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
u16 i; u16 i;
encode = vma->vm->scratch_pte; encode = vma->vm->scratch_pte;
vaddr = kmap_atomic_px(pd->page_table[idx.pde]); vaddr = kmap_atomic_px(i915_pt_entry(pd,
idx.pde));
for (i = 1; i < index; i += 16) for (i = 1; i < index; i += 16)
memset64(vaddr + i, encode, 15); memset64(vaddr + i, encode, 15);
@ -1195,15 +1183,16 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
{ {
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
struct sgt_dma iter = sgt_dma(vma); struct sgt_dma iter = sgt_dma(vma);
struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps; struct i915_page_directory * const pml4 = ppgtt->pd;
if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level, gen8_ppgtt_insert_huge_entries(vma, pml4, &iter, cache_level,
flags); flags);
} else { } else {
struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start); struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++], while (gen8_ppgtt_insert_pte_entries(ppgtt,
i915_pdp_entry(pml4, idx.pml4e++),
&iter, &idx, cache_level, &iter, &idx, cache_level,
flags)) flags))
GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4); GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
@ -1218,8 +1207,8 @@ static void gen8_free_page_tables(struct i915_address_space *vm,
int i; int i;
for (i = 0; i < I915_PDES; i++) { for (i = 0; i < I915_PDES; i++) {
if (pd->page_table[i] != vm->scratch_pt) if (pd->entry[i] != vm->scratch_pt)
free_pt(vm, pd->page_table[i]); free_pt(vm, pd->entry[i]);
} }
} }
@ -1278,7 +1267,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
gen8_initialize_pt(vm, vm->scratch_pt); gen8_initialize_pt(vm, vm->scratch_pt);
gen8_initialize_pd(vm, vm->scratch_pd); gen8_initialize_pd(vm, vm->scratch_pd);
if (i915_vm_is_4lvl(vm)) if (i915_vm_is_4lvl(vm))
gen8_initialize_pdp(vm, vm->scratch_pdp); gen8_initialize_4lvl_pdp(vm, vm->scratch_pdp);
return 0; return 0;
@ -1300,7 +1289,7 @@ static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
int i; int i;
if (i915_vm_is_4lvl(vm)) { if (i915_vm_is_4lvl(vm)) {
const u64 daddr = px_dma(&ppgtt->pml4); const u64 daddr = px_dma(ppgtt->pd);
I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr)); I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr)); I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
@ -1337,17 +1326,17 @@ static void gen8_free_scratch(struct i915_address_space *vm)
} }
static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm, static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
struct i915_page_directory_pointer *pdp) struct i915_page_directory *pdp)
{ {
const unsigned int pdpes = i915_pdpes_per_pdp(vm); const unsigned int pdpes = i915_pdpes_per_pdp(vm);
int i; int i;
for (i = 0; i < pdpes; i++) { for (i = 0; i < pdpes; i++) {
if (pdp->page_directory[i] == vm->scratch_pd) if (pdp->entry[i] == vm->scratch_pd)
continue; continue;
gen8_free_page_tables(vm, pdp->page_directory[i]); gen8_free_page_tables(vm, pdp->entry[i]);
free_pd(vm, pdp->page_directory[i]); free_pd(vm, pdp->entry[i]);
} }
free_pdp(vm, pdp); free_pdp(vm, pdp);
@ -1355,16 +1344,19 @@ static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt) static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
{ {
struct i915_page_directory * const pml4 = ppgtt->pd;
int i; int i;
for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) { for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp) struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);
if (pdp == ppgtt->vm.scratch_pdp)
continue; continue;
gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]); gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);
} }
cleanup_px(&ppgtt->vm, &ppgtt->pml4); free_pd(&ppgtt->vm, pml4);
} }
static void gen8_ppgtt_cleanup(struct i915_address_space *vm) static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
@ -1378,7 +1370,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
if (i915_vm_is_4lvl(vm)) if (i915_vm_is_4lvl(vm))
gen8_ppgtt_cleanup_4lvl(ppgtt); gen8_ppgtt_cleanup_4lvl(ppgtt);
else else
gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp); gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);
gen8_free_scratch(vm); gen8_free_scratch(vm);
} }
@ -1407,10 +1399,10 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
if (count < GEN8_PTES || intel_vgpu_active(vm->i915)) if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
gen8_initialize_pt(vm, pt); gen8_initialize_pt(vm, pt);
old = cmpxchg(&pd->page_table[pde], vm->scratch_pt, pt); old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt);
if (old == vm->scratch_pt) { if (old == vm->scratch_pt) {
gen8_ppgtt_set_pde(vm, pd, pt, pde); gen8_ppgtt_set_pde(vm, pd, pt, pde);
atomic_inc(&pd->used_pdes); atomic_inc(&pd->used);
} else { } else {
free_pt(vm, pt); free_pt(vm, pt);
pt = old; pt = old;
@ -1419,7 +1411,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
spin_lock(&pd->lock); spin_lock(&pd->lock);
} }
atomic_add(count, &pt->used_ptes); atomic_add(count, &pt->used);
} }
spin_unlock(&pd->lock); spin_unlock(&pd->lock);
@ -1431,7 +1423,7 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
} }
static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm, static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
struct i915_page_directory_pointer *pdp, struct i915_page_directory *pdp,
u64 start, u64 length) u64 start, u64 length)
{ {
struct i915_page_directory *pd; struct i915_page_directory *pd;
@ -1452,11 +1444,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
gen8_initialize_pd(vm, pd); gen8_initialize_pd(vm, pd);
old = cmpxchg(&pdp->page_directory[pdpe], old = cmpxchg(&pdp->entry[pdpe], vm->scratch_pd, pd);
vm->scratch_pd, pd);
if (old == vm->scratch_pd) { if (old == vm->scratch_pd) {
gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
atomic_inc(&pdp->used_pdpes); atomic_inc(&pdp->used);
} else { } else {
free_pd(vm, pd); free_pd(vm, pd);
pd = old; pd = old;
@ -1464,7 +1455,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
spin_lock(&pdp->lock); spin_lock(&pdp->lock);
} }
atomic_inc(&pd->used_pdes); atomic_inc(&pd->used);
spin_unlock(&pdp->lock); spin_unlock(&pdp->lock);
ret = gen8_ppgtt_alloc_pd(vm, pd, start, length); ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
@ -1472,7 +1463,7 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
goto unwind_pd; goto unwind_pd;
spin_lock(&pdp->lock); spin_lock(&pdp->lock);
atomic_dec(&pd->used_pdes); atomic_dec(&pd->used);
} }
spin_unlock(&pdp->lock); spin_unlock(&pdp->lock);
@ -1480,10 +1471,10 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
unwind_pd: unwind_pd:
spin_lock(&pdp->lock); spin_lock(&pdp->lock);
if (atomic_dec_and_test(&pd->used_pdes)) { if (atomic_dec_and_test(&pd->used)) {
gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
GEM_BUG_ON(!atomic_read(&pdp->used_pdpes)); GEM_BUG_ON(!atomic_read(&pdp->used));
atomic_dec(&pdp->used_pdpes); atomic_dec(&pdp->used);
free_pd(vm, pd); free_pd(vm, pd);
} }
spin_unlock(&pdp->lock); spin_unlock(&pdp->lock);
@ -1496,23 +1487,24 @@ static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
u64 start, u64 length) u64 start, u64 length)
{ {
return gen8_ppgtt_alloc_pdp(vm, return gen8_ppgtt_alloc_pdp(vm,
&i915_vm_to_ppgtt(vm)->pdp, start, length); i915_vm_to_ppgtt(vm)->pd, start, length);
} }
static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm, static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
u64 start, u64 length) u64 start, u64 length)
{ {
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
struct i915_pml4 *pml4 = &ppgtt->pml4; struct i915_page_directory * const pml4 = ppgtt->pd;
struct i915_page_directory_pointer *pdp; struct i915_page_directory *pdp;
u64 from = start; u64 from = start;
u32 pml4e; u32 pml4e;
int ret; int ret;
spin_lock(&pml4->lock); spin_lock(&pml4->lock);
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
if (pdp == vm->scratch_pdp) { if (pdp == vm->scratch_pdp) {
struct i915_page_directory_pointer *old; struct i915_page_directory *old;
spin_unlock(&pml4->lock); spin_unlock(&pml4->lock);
@ -1520,9 +1512,9 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
if (IS_ERR(pdp)) if (IS_ERR(pdp))
goto unwind; goto unwind;
gen8_initialize_pdp(vm, pdp); gen8_initialize_4lvl_pdp(vm, pdp);
old = cmpxchg(&pml4->pdps[pml4e], vm->scratch_pdp, pdp); old = cmpxchg(&pml4->entry[pml4e], vm->scratch_pdp, pdp);
if (old == vm->scratch_pdp) { if (old == vm->scratch_pdp) {
gen8_ppgtt_set_pml4e(pml4, pdp, pml4e); gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
} else { } else {
@ -1532,7 +1524,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
spin_lock(&pml4->lock); spin_lock(&pml4->lock);
} }
atomic_inc(&pdp->used_pdpes); atomic_inc(&pdp->used);
spin_unlock(&pml4->lock); spin_unlock(&pml4->lock);
ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length); ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
@ -1540,7 +1532,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
goto unwind_pdp; goto unwind_pdp;
spin_lock(&pml4->lock); spin_lock(&pml4->lock);
atomic_dec(&pdp->used_pdpes); atomic_dec(&pdp->used);
} }
spin_unlock(&pml4->lock); spin_unlock(&pml4->lock);
@ -1548,7 +1540,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
unwind_pdp: unwind_pdp:
spin_lock(&pml4->lock); spin_lock(&pml4->lock);
if (atomic_dec_and_test(&pdp->used_pdpes)) { if (atomic_dec_and_test(&pdp->used)) {
gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e); gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
free_pdp(vm, pdp); free_pdp(vm, pdp);
} }
@ -1561,7 +1553,7 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
{ {
struct i915_address_space *vm = &ppgtt->vm; struct i915_address_space *vm = &ppgtt->vm;
struct i915_page_directory_pointer *pdp = &ppgtt->pdp; struct i915_page_directory *pdp = ppgtt->pd;
struct i915_page_directory *pd; struct i915_page_directory *pd;
u64 start = 0, length = ppgtt->vm.total; u64 start = 0, length = ppgtt->vm.total;
u64 from = start; u64 from = start;
@ -1574,10 +1566,12 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
gen8_initialize_pd(vm, pd); gen8_initialize_pd(vm, pd);
gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe); gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
atomic_inc(&pdp->used_pdpes);
atomic_inc(&pdp->used);
} }
atomic_inc(&pdp->used_pdpes); /* never remove */ atomic_inc(&pdp->used); /* never remove */
return 0; return 0;
unwind: unwind:
@ -1586,7 +1580,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe); gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
free_pd(vm, pd); free_pd(vm, pd);
} }
atomic_set(&pdp->used_pdpes, 0); atomic_set(&pdp->used, 0);
return -ENOMEM; return -ENOMEM;
} }
@ -1641,27 +1635,25 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
if (err) if (err)
goto err_free; goto err_free;
if (i915_vm_is_4lvl(&ppgtt->vm)) { ppgtt->pd = alloc_pdp(&ppgtt->vm);
err = setup_px(&ppgtt->vm, &ppgtt->pml4); if (IS_ERR(ppgtt->pd)) {
if (err) err = PTR_ERR(ppgtt->pd);
goto err_scratch; goto err_scratch;
}
gen8_initialize_pml4(&ppgtt->vm, &ppgtt->pml4); if (i915_vm_is_4lvl(&ppgtt->vm)) {
gen8_initialize_pml4(&ppgtt->vm, ppgtt->pd);
ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl; ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;
ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl; ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;
ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl; ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl;
} else { } else {
err = __pdp_init(&ppgtt->vm, &ppgtt->pdp); gen8_initialize_3lvl_pdp(&ppgtt->vm, ppgtt->pd);
if (err)
goto err_scratch;
if (intel_vgpu_active(i915)) { if (intel_vgpu_active(i915)) {
err = gen8_preallocate_top_level_pdp(ppgtt); err = gen8_preallocate_top_level_pdp(ppgtt);
if (err) { if (err)
__pdp_fini(&ppgtt->pdp); goto err_pdp;
goto err_scratch;
}
} }
ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl; ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;
@ -1676,6 +1668,8 @@ static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
return ppgtt; return ppgtt;
err_pdp:
free_pdp(&ppgtt->vm, ppgtt->pd);
err_scratch: err_scratch:
gen8_free_scratch(&ppgtt->vm); gen8_free_scratch(&ppgtt->vm);
err_free: err_free:
@ -1741,15 +1735,16 @@ static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
static void gen6_ppgtt_clear_range(struct i915_address_space *vm, static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
u64 start, u64 length) u64 start, u64 length)
{ {
struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
unsigned int first_entry = start / I915_GTT_PAGE_SIZE; const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
const gen6_pte_t scratch_pte = vm->scratch_pte;
unsigned int pde = first_entry / GEN6_PTES; unsigned int pde = first_entry / GEN6_PTES;
unsigned int pte = first_entry % GEN6_PTES; unsigned int pte = first_entry % GEN6_PTES;
unsigned int num_entries = length / I915_GTT_PAGE_SIZE; unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
const gen6_pte_t scratch_pte = vm->scratch_pte;
while (num_entries) { while (num_entries) {
struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++]; struct i915_page_table * const pt =
i915_pt_entry(ppgtt->base.pd, pde++);
const unsigned int count = min(num_entries, GEN6_PTES - pte); const unsigned int count = min(num_entries, GEN6_PTES - pte);
gen6_pte_t *vaddr; gen6_pte_t *vaddr;
@ -1757,8 +1752,8 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
num_entries -= count; num_entries -= count;
GEM_BUG_ON(count > atomic_read(&pt->used_ptes)); GEM_BUG_ON(count > atomic_read(&pt->used));
if (!atomic_sub_return(count, &pt->used_ptes)) if (!atomic_sub_return(count, &pt->used))
ppgtt->scan_for_unused_pt = true; ppgtt->scan_for_unused_pt = true;
/* /*
@ -1782,6 +1777,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
u32 flags) u32 flags)
{ {
struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
struct i915_page_directory * const pd = ppgtt->pd;
unsigned first_entry = vma->node.start / I915_GTT_PAGE_SIZE; unsigned first_entry = vma->node.start / I915_GTT_PAGE_SIZE;
unsigned act_pt = first_entry / GEN6_PTES; unsigned act_pt = first_entry / GEN6_PTES;
unsigned act_pte = first_entry % GEN6_PTES; unsigned act_pte = first_entry % GEN6_PTES;
@ -1789,9 +1785,9 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
struct sgt_dma iter = sgt_dma(vma); struct sgt_dma iter = sgt_dma(vma);
gen6_pte_t *vaddr; gen6_pte_t *vaddr;
GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt); GEM_BUG_ON(i915_pt_entry(pd, act_pt) == vm->scratch_pt);
vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]); vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));
do { do {
vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
@ -1807,7 +1803,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
if (++act_pte == GEN6_PTES) { if (++act_pte == GEN6_PTES) {
kunmap_atomic(vaddr); kunmap_atomic(vaddr);
vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]); vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt));
act_pte = 0; act_pte = 0;
} }
} while (1); } while (1);
@ -1820,6 +1816,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
u64 start, u64 length) u64 start, u64 length)
{ {
struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm)); struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
struct i915_page_directory * const pd = ppgtt->base.pd;
struct i915_page_table *pt; struct i915_page_table *pt;
intel_wakeref_t wakeref; intel_wakeref_t wakeref;
u64 from = start; u64 from = start;
@ -1828,14 +1825,14 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
spin_lock(&ppgtt->base.pd.lock); spin_lock(&pd->lock);
gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) { gen6_for_each_pde(pt, pd, start, length, pde) {
const unsigned int count = gen6_pte_count(start, length); const unsigned int count = gen6_pte_count(start, length);
if (pt == vm->scratch_pt) { if (pt == vm->scratch_pt) {
struct i915_page_table *old; struct i915_page_table *old;
spin_unlock(&ppgtt->base.pd.lock); spin_unlock(&pd->lock);
pt = alloc_pt(vm); pt = alloc_pt(vm);
if (IS_ERR(pt)) if (IS_ERR(pt))
@ -1843,10 +1840,8 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
gen6_initialize_pt(vm, pt); gen6_initialize_pt(vm, pt);
old = cmpxchg(&ppgtt->base.pd.page_table[pde], old = cmpxchg(&pd->entry[pde], vm->scratch_pt, pt);
vm->scratch_pt, pt);
if (old == vm->scratch_pt) { if (old == vm->scratch_pt) {
ppgtt->base.pd.page_table[pde] = pt;
if (i915_vma_is_bound(ppgtt->vma, if (i915_vma_is_bound(ppgtt->vma,
I915_VMA_GLOBAL_BIND)) { I915_VMA_GLOBAL_BIND)) {
gen6_write_pde(ppgtt, pde, pt); gen6_write_pde(ppgtt, pde, pt);
@ -1857,12 +1852,12 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
pt = old; pt = old;
} }
spin_lock(&ppgtt->base.pd.lock); spin_lock(&pd->lock);
} }
atomic_add(count, &pt->used_ptes); atomic_add(count, &pt->used);
} }
spin_unlock(&ppgtt->base.pd.lock); spin_unlock(&pd->lock);
if (flush) { if (flush) {
mark_tlbs_dirty(&ppgtt->base); mark_tlbs_dirty(&ppgtt->base);
@ -1882,6 +1877,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt) static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
{ {
struct i915_address_space * const vm = &ppgtt->base.vm; struct i915_address_space * const vm = &ppgtt->base.vm;
struct i915_page_directory * const pd = ppgtt->base.pd;
struct i915_page_table *unused; struct i915_page_table *unused;
u32 pde; u32 pde;
int ret; int ret;
@ -1901,9 +1897,9 @@ static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
} }
gen6_initialize_pt(vm, vm->scratch_pt); gen6_initialize_pt(vm, vm->scratch_pt);
gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)
ppgtt->base.pd.page_table[pde] = vm->scratch_pt; gen6_for_all_pdes(unused, pd, pde)
spin_lock_init(&ppgtt->base.pd.lock); pd->entry[pde] = vm->scratch_pt;
return 0; return 0;
} }
@ -1916,10 +1912,11 @@ static void gen6_ppgtt_free_scratch(struct i915_address_space *vm)
static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt) static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
{ {
struct i915_page_directory * const pd = ppgtt->base.pd;
struct i915_page_table *pt; struct i915_page_table *pt;
u32 pde; u32 pde;
gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) gen6_for_all_pdes(pt, pd, pde)
if (pt != ppgtt->base.vm.scratch_pt) if (pt != ppgtt->base.vm.scratch_pt)
free_pt(&ppgtt->base.vm, pt); free_pt(&ppgtt->base.vm, pt);
} }
@ -1983,6 +1980,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
gen6_ppgtt_free_pd(ppgtt); gen6_ppgtt_free_pd(ppgtt);
gen6_ppgtt_free_scratch(vm); gen6_ppgtt_free_scratch(vm);
kfree(ppgtt->base.pd);
} }
static int pd_vma_set_pages(struct i915_vma *vma) static int pd_vma_set_pages(struct i915_vma *vma)
@ -2008,10 +2006,10 @@ static int pd_vma_bind(struct i915_vma *vma,
struct i915_page_table *pt; struct i915_page_table *pt;
unsigned int pde; unsigned int pde;
ppgtt->base.pd.base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t); ppgtt->base.pd->base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);
ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset; ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;
gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) gen6_for_all_pdes(pt, ppgtt->base.pd, pde)
gen6_write_pde(ppgtt, pde, pt); gen6_write_pde(ppgtt, pde, pt);
mark_tlbs_dirty(&ppgtt->base); mark_tlbs_dirty(&ppgtt->base);
@ -2023,6 +2021,7 @@ static int pd_vma_bind(struct i915_vma *vma,
static void pd_vma_unbind(struct i915_vma *vma) static void pd_vma_unbind(struct i915_vma *vma)
{ {
struct gen6_ppgtt *ppgtt = vma->private; struct gen6_ppgtt *ppgtt = vma->private;
struct i915_page_directory * const pd = ppgtt->base.pd;
struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt; struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt;
struct i915_page_table *pt; struct i915_page_table *pt;
unsigned int pde; unsigned int pde;
@ -2031,12 +2030,12 @@ static void pd_vma_unbind(struct i915_vma *vma)
return; return;
/* Free all no longer used page tables */ /* Free all no longer used page tables */
gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) { gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
if (atomic_read(&pt->used_ptes) || pt == scratch_pt) if (atomic_read(&pt->used) || pt == scratch_pt)
continue; continue;
free_pt(&ppgtt->base.vm, pt); free_pt(&ppgtt->base.vm, pt);
ppgtt->base.pd.page_table[pde] = scratch_pt; pd->entry[pde] = scratch_pt;
} }
ppgtt->scan_for_unused_pt = false; ppgtt->scan_for_unused_pt = false;
@ -2165,9 +2164,15 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
goto err_free; goto err_free;
} }
ppgtt->base.pd = __alloc_pd();
if (!ppgtt->base.pd) {
err = -ENOMEM;
goto err_work;
}
err = gen6_ppgtt_init_scratch(ppgtt); err = gen6_ppgtt_init_scratch(ppgtt);
if (err) if (err)
goto err_work; goto err_pd;
ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE); ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);
if (IS_ERR(ppgtt->vma)) { if (IS_ERR(ppgtt->vma)) {
@ -2179,6 +2184,8 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
err_scratch: err_scratch:
gen6_ppgtt_free_scratch(&ppgtt->base.vm); gen6_ppgtt_free_scratch(&ppgtt->base.vm);
err_pd:
kfree(ppgtt->base.pd);
err_work: err_work:
kfree(ppgtt->work); kfree(ppgtt->work);
err_free: err_free:

64
drivers/gpu/drm/i915/i915_gem_gtt.h
View File

@ -248,28 +248,14 @@ struct i915_page_dma {
struct i915_page_table { struct i915_page_table {
struct i915_page_dma base; struct i915_page_dma base;
atomic_t used_ptes; atomic_t used;
}; };
struct i915_page_directory { struct i915_page_directory {
struct i915_page_dma base; struct i915_page_dma base;
atomic_t used;
struct i915_page_table *page_table[I915_PDES]; /* PDEs */
atomic_t used_pdes;
spinlock_t lock;
};
struct i915_page_directory_pointer {
struct i915_page_dma base;
struct i915_page_directory **page_directory;
atomic_t used_pdpes;
spinlock_t lock;
};
struct i915_pml4 {
struct i915_page_dma base;
struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4];
spinlock_t lock; spinlock_t lock;
void *entry[512];
}; };
struct i915_vma_ops { struct i915_vma_ops {
@ -321,7 +307,7 @@ struct i915_address_space {
struct i915_page_dma scratch_page; struct i915_page_dma scratch_page;
struct i915_page_table *scratch_pt; struct i915_page_table *scratch_pt;
struct i915_page_directory *scratch_pd; struct i915_page_directory *scratch_pd;
struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */ struct i915_page_directory *scratch_pdp; /* GEN8+ & 48b PPGTT */
/** /**
* List of vma currently bound. * List of vma currently bound.
@ -428,11 +414,7 @@ struct i915_ppgtt {
struct i915_address_space vm; struct i915_address_space vm;
intel_engine_mask_t pd_dirty_engines; intel_engine_mask_t pd_dirty_engines;
union { struct i915_page_directory *pd;
struct i915_pml4 pml4; /* GEN8+ & 48b PPGTT */
struct i915_page_directory_pointer pdp; /* GEN8+ */
struct i915_page_directory pd; /* GEN6-7 */
};
}; };
struct gen6_ppgtt { struct gen6_ppgtt {
@ -466,7 +448,7 @@ static inline struct gen6_ppgtt *to_gen6_ppgtt(struct i915_ppgtt *base)
#define gen6_for_each_pde(pt, pd, start, length, iter) \ #define gen6_for_each_pde(pt, pd, start, length, iter) \
for (iter = gen6_pde_index(start); \ for (iter = gen6_pde_index(start); \
length > 0 && iter < I915_PDES && \ length > 0 && iter < I915_PDES && \
(pt = (pd)->page_table[iter], true); \ (pt = i915_pt_entry(pd, iter), true); \
({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT); \ ({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT); \
temp = min(temp - start, length); \ temp = min(temp - start, length); \
start += temp, length -= temp; }), ++iter) start += temp, length -= temp; }), ++iter)
@ -474,7 +456,7 @@ static inline struct gen6_ppgtt *to_gen6_ppgtt(struct i915_ppgtt *base)
#define gen6_for_all_pdes(pt, pd, iter) \ #define gen6_for_all_pdes(pt, pd, iter) \
for (iter = 0; \ for (iter = 0; \
iter < I915_PDES && \ iter < I915_PDES && \
(pt = (pd)->page_table[iter], true); \ (pt = i915_pt_entry(pd, iter), true); \
++iter) ++iter)
static inline u32 i915_pte_index(u64 address, unsigned int pde_shift) static inline u32 i915_pte_index(u64 address, unsigned int pde_shift)
@ -533,6 +515,27 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm)
return GEN8_3LVL_PDPES; return GEN8_3LVL_PDPES;
} }
static inline struct i915_page_table *
i915_pt_entry(const struct i915_page_directory * const pd,
const unsigned short n)
{
return pd->entry[n];
}
static inline struct i915_page_directory *
i915_pd_entry(const struct i915_page_directory * const pdp,
const unsigned short n)
{
return pdp->entry[n];
}
static inline struct i915_page_directory *
i915_pdp_entry(const struct i915_page_directory * const pml4,
const unsigned short n)
{
return pml4->entry[n];
}
/* Equivalent to the gen6 version, For each pde iterates over every pde /* Equivalent to the gen6 version, For each pde iterates over every pde
* between from start until start + length. On gen8+ it simply iterates * between from start until start + length. On gen8+ it simply iterates
* over every page directory entry in a page directory. * over every page directory entry in a page directory.
@ -540,7 +543,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm)
#define gen8_for_each_pde(pt, pd, start, length, iter) \ #define gen8_for_each_pde(pt, pd, start, length, iter) \
for (iter = gen8_pde_index(start); \ for (iter = gen8_pde_index(start); \
length > 0 && iter < I915_PDES && \ length > 0 && iter < I915_PDES && \
(pt = (pd)->page_table[iter], true); \ (pt = i915_pt_entry(pd, iter), true); \
({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT); \ ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT); \
temp = min(temp - start, length); \ temp = min(temp - start, length); \
start += temp, length -= temp; }), ++iter) start += temp, length -= temp; }), ++iter)
@ -548,7 +551,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm)
#define gen8_for_each_pdpe(pd, pdp, start, length, iter) \ #define gen8_for_each_pdpe(pd, pdp, start, length, iter) \
for (iter = gen8_pdpe_index(start); \ for (iter = gen8_pdpe_index(start); \
length > 0 && iter < i915_pdpes_per_pdp(vm) && \ length > 0 && iter < i915_pdpes_per_pdp(vm) && \
(pd = (pdp)->page_directory[iter], true); \ (pd = i915_pd_entry(pdp, iter), true); \
({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT); \ ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT); \
temp = min(temp - start, length); \ temp = min(temp - start, length); \
start += temp, length -= temp; }), ++iter) start += temp, length -= temp; }), ++iter)
@ -556,7 +559,7 @@ i915_pdpes_per_pdp(const struct i915_address_space *vm)
#define gen8_for_each_pml4e(pdp, pml4, start, length, iter) \ #define gen8_for_each_pml4e(pdp, pml4, start, length, iter) \
for (iter = gen8_pml4e_index(start); \ for (iter = gen8_pml4e_index(start); \
length > 0 && iter < GEN8_PML4ES_PER_PML4 && \ length > 0 && iter < GEN8_PML4ES_PER_PML4 && \
(pdp = (pml4)->pdps[iter], true); \ (pdp = i915_pdp_entry(pml4, iter), true); \
({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT); \ ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT); \
temp = min(temp - start, length); \ temp = min(temp - start, length); \
start += temp, length -= temp; }), ++iter) start += temp, length -= temp; }), ++iter)
@ -589,7 +592,10 @@ static inline u64 gen8_pte_count(u64 address, u64 length)
static inline dma_addr_t static inline dma_addr_t
i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n) i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n)
{ {
return px_dma(ppgtt->pdp.page_directory[n]); struct i915_page_directory *pd;
pd = i915_pdp_entry(ppgtt->pd, n);
return px_dma(pd);
} }
static inline struct i915_ggtt * static inline struct i915_ggtt *