mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-25 14:33:30 +07:00
2334b75ffb
In TTM world the pages for the graphic drivers are kept in three different pools: write combined, uncached, and cached (write-back). When the pages are used by the graphic driver the graphic adapter via its built in MMU (or AGP) programs these pages in. The programming requires the virtual address (from the graphic adapter perspective) and the physical address (either System RAM or the memory on the card) which is obtained using the pci_map_* calls (which does the virtual to physical - or bus address translation). During the graphic application's "life" those pages can be shuffled around, swapped out to disk, moved from the VRAM to System RAM or vice-versa. This all works with the existing TTM pool code - except when we want to use the software IOTLB (SWIOTLB) code to "map" the physical addresses to the graphic adapter MMU. We end up programming the bounce buffer's physical address instead of the TTM pool memory's and get a non-worky driver. There are two solutions: 1) using the DMA API to allocate pages that are screened by the DMA API, or 2) using the pci_sync_* calls to copy the pages from the bounce-buffer and back. This patch fixes the issue by allocating pages using the DMA API. The second is a viable option - but it has performance drawbacks and potential correctness issues - think of the write cache page being bounced (SWIOTLB->TTM), the WC is set on the TTM page and the copy from SWIOTLB not making it to the TTM page until the page has been recycled in the pool (and used by another application). The bounce buffer does not get activated often - only in cases where we have a 32-bit capable card and we want to use a page that is allocated above the 4GB limit. The bounce buffer offers the solution of copying the contents of that 4GB page to an location below 4GB and then back when the operation has been completed (or vice-versa). This is done by using the 'pci_sync_*' calls. Note: If you look carefully enough in the existing TTM page pool code you will notice the GFP_DMA32 flag is used - which should guarantee that the provided page is under 4GB. It certainly is the case, except this gets ignored in two cases: - If user specifies 'swiotlb=force' which bounces _every_ page. - If user is using a Xen's PV Linux guest (which uses the SWIOTLB and the underlaying PFN's aren't necessarily under 4GB). To not have this extra copying done the other option is to allocate the pages using the DMA API so that there is not need to map the page and perform the expensive 'pci_sync_*' calls. This DMA API capable TTM pool requires for this the 'struct device' to properly call the DMA API. It also has to track the virtual and bus address of the page being handed out in case it ends up being swapped out or de-allocated - to make sure it is de-allocated using the proper's 'struct device'. Implementation wise the code keeps two lists: one that is attached to the 'struct device' (via the dev->dma_pools list) and a global one to be used when the 'struct device' is unavailable (think shrinker code). The global list can iterate over all of the 'struct device' and its associated dma_pool. The list in dev->dma_pools can only iterate the device's dma_pool. /[struct device_pool]\ /---------------------------------------------------| dev | / +-------| dma_pool | /-----+------\ / \--------------------/ |struct device| /-->[struct dma_pool for WC]</ /[struct device_pool]\ | dma_pools +----+ /-| dev | | ... | \--->[struct dma_pool for uncached]<-/--| dma_pool | \-----+------/ / \--------------------/ \----------------------------------------------/ [Two pools associated with the device (WC and UC), and the parallel list containing the 'struct dev' and 'struct dma_pool' entries] The maximum amount of dma pools a device can have is six: write-combined, uncached, and cached; then there are the DMA32 variants which are: write-combined dma32, uncached dma32, and cached dma32. Currently this code only gets activated when any variant of the SWIOTLB IOMMU code is running (Intel without VT-d, AMD without GART, IBM Calgary and Xen PV with PCI devices). Tested-by: Michel Dänzer <michel@daenzer.net> [v1: Using swiotlb_nr_tbl instead of swiotlb_enabled] [v2: Major overhaul - added 'inuse_list' to seperate used from inuse and reorder the order of lists to get better performance.] [v3: Added comments/and some logic based on review, Added Jerome tag] [v4: rebase on top of ttm_tt & ttm_backend merge] [v5: rebase on top of ttm memory accounting overhaul] [v6: New rebase on top of more memory accouting changes] [v7: well rebase on top of no memory accounting changes] [v8: make sure pages list is initialized empty] [v9: calll ttm_mem_global_free_page in unpopulate for accurate accountg] Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Reviewed-by: Jerome Glisse <jglisse@redhat.com> Acked-by: Thomas Hellstrom <thellstrom@vmware.com>
352 lines
8.5 KiB
C
352 lines
8.5 KiB
C
/**************************************************************************
|
|
*
|
|
* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
|
|
* All Rights Reserved.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the
|
|
* "Software"), to deal in the Software without restriction, including
|
|
* without limitation the rights to use, copy, modify, merge, publish,
|
|
* distribute, sub license, and/or sell copies of the Software, and to
|
|
* permit persons to whom the Software is furnished to do so, subject to
|
|
* the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice (including the
|
|
* next paragraph) shall be included in all copies or substantial portions
|
|
* of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
|
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
|
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
*
|
|
**************************************************************************/
|
|
/*
|
|
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
|
|
*/
|
|
|
|
#include <linux/sched.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/shmem_fs.h>
|
|
#include <linux/file.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/export.h>
|
|
#include "drm_cache.h"
|
|
#include "drm_mem_util.h"
|
|
#include "ttm/ttm_module.h"
|
|
#include "ttm/ttm_bo_driver.h"
|
|
#include "ttm/ttm_placement.h"
|
|
#include "ttm/ttm_page_alloc.h"
|
|
|
|
/**
|
|
* Allocates storage for pointers to the pages that back the ttm.
|
|
*/
|
|
static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
|
|
{
|
|
ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(*ttm->pages));
|
|
ttm->dma_address = drm_calloc_large(ttm->num_pages,
|
|
sizeof(*ttm->dma_address));
|
|
}
|
|
|
|
static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
|
|
{
|
|
drm_free_large(ttm->pages);
|
|
ttm->pages = NULL;
|
|
drm_free_large(ttm->dma_address);
|
|
ttm->dma_address = NULL;
|
|
}
|
|
|
|
#ifdef CONFIG_X86
|
|
static inline int ttm_tt_set_page_caching(struct page *p,
|
|
enum ttm_caching_state c_old,
|
|
enum ttm_caching_state c_new)
|
|
{
|
|
int ret = 0;
|
|
|
|
if (PageHighMem(p))
|
|
return 0;
|
|
|
|
if (c_old != tt_cached) {
|
|
/* p isn't in the default caching state, set it to
|
|
* writeback first to free its current memtype. */
|
|
|
|
ret = set_pages_wb(p, 1);
|
|
if (ret)
|
|
return ret;
|
|
}
|
|
|
|
if (c_new == tt_wc)
|
|
ret = set_memory_wc((unsigned long) page_address(p), 1);
|
|
else if (c_new == tt_uncached)
|
|
ret = set_pages_uc(p, 1);
|
|
|
|
return ret;
|
|
}
|
|
#else /* CONFIG_X86 */
|
|
static inline int ttm_tt_set_page_caching(struct page *p,
|
|
enum ttm_caching_state c_old,
|
|
enum ttm_caching_state c_new)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_X86 */
|
|
|
|
/*
|
|
* Change caching policy for the linear kernel map
|
|
* for range of pages in a ttm.
|
|
*/
|
|
|
|
static int ttm_tt_set_caching(struct ttm_tt *ttm,
|
|
enum ttm_caching_state c_state)
|
|
{
|
|
int i, j;
|
|
struct page *cur_page;
|
|
int ret;
|
|
|
|
if (ttm->caching_state == c_state)
|
|
return 0;
|
|
|
|
if (ttm->state == tt_unpopulated) {
|
|
/* Change caching but don't populate */
|
|
ttm->caching_state = c_state;
|
|
return 0;
|
|
}
|
|
|
|
if (ttm->caching_state == tt_cached)
|
|
drm_clflush_pages(ttm->pages, ttm->num_pages);
|
|
|
|
for (i = 0; i < ttm->num_pages; ++i) {
|
|
cur_page = ttm->pages[i];
|
|
if (likely(cur_page != NULL)) {
|
|
ret = ttm_tt_set_page_caching(cur_page,
|
|
ttm->caching_state,
|
|
c_state);
|
|
if (unlikely(ret != 0))
|
|
goto out_err;
|
|
}
|
|
}
|
|
|
|
ttm->caching_state = c_state;
|
|
|
|
return 0;
|
|
|
|
out_err:
|
|
for (j = 0; j < i; ++j) {
|
|
cur_page = ttm->pages[j];
|
|
if (likely(cur_page != NULL)) {
|
|
(void)ttm_tt_set_page_caching(cur_page, c_state,
|
|
ttm->caching_state);
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)
|
|
{
|
|
enum ttm_caching_state state;
|
|
|
|
if (placement & TTM_PL_FLAG_WC)
|
|
state = tt_wc;
|
|
else if (placement & TTM_PL_FLAG_UNCACHED)
|
|
state = tt_uncached;
|
|
else
|
|
state = tt_cached;
|
|
|
|
return ttm_tt_set_caching(ttm, state);
|
|
}
|
|
EXPORT_SYMBOL(ttm_tt_set_placement_caching);
|
|
|
|
void ttm_tt_destroy(struct ttm_tt *ttm)
|
|
{
|
|
if (unlikely(ttm == NULL))
|
|
return;
|
|
|
|
if (ttm->state == tt_bound) {
|
|
ttm_tt_unbind(ttm);
|
|
}
|
|
|
|
if (likely(ttm->pages != NULL)) {
|
|
ttm->bdev->driver->ttm_tt_unpopulate(ttm);
|
|
ttm_tt_free_page_directory(ttm);
|
|
}
|
|
|
|
if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&
|
|
ttm->swap_storage)
|
|
fput(ttm->swap_storage);
|
|
|
|
ttm->swap_storage = NULL;
|
|
ttm->func->destroy(ttm);
|
|
}
|
|
|
|
int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev,
|
|
unsigned long size, uint32_t page_flags,
|
|
struct page *dummy_read_page)
|
|
{
|
|
ttm->bdev = bdev;
|
|
ttm->glob = bdev->glob;
|
|
ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
|
|
ttm->caching_state = tt_cached;
|
|
ttm->page_flags = page_flags;
|
|
ttm->dummy_read_page = dummy_read_page;
|
|
ttm->state = tt_unpopulated;
|
|
|
|
INIT_LIST_HEAD(&ttm->alloc_list);
|
|
ttm_tt_alloc_page_directory(ttm);
|
|
if (!ttm->pages || !ttm->dma_address) {
|
|
ttm_tt_destroy(ttm);
|
|
printk(KERN_ERR TTM_PFX "Failed allocating page table\n");
|
|
return -ENOMEM;
|
|
}
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(ttm_tt_init);
|
|
|
|
void ttm_tt_unbind(struct ttm_tt *ttm)
|
|
{
|
|
int ret;
|
|
|
|
if (ttm->state == tt_bound) {
|
|
ret = ttm->func->unbind(ttm);
|
|
BUG_ON(ret);
|
|
ttm->state = tt_unbound;
|
|
}
|
|
}
|
|
|
|
int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
|
|
{
|
|
int ret = 0;
|
|
|
|
if (!ttm)
|
|
return -EINVAL;
|
|
|
|
if (ttm->state == tt_bound)
|
|
return 0;
|
|
|
|
ret = ttm->bdev->driver->ttm_tt_populate(ttm);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = ttm->func->bind(ttm, bo_mem);
|
|
if (unlikely(ret != 0))
|
|
return ret;
|
|
|
|
ttm->state = tt_bound;
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(ttm_tt_bind);
|
|
|
|
int ttm_tt_swapin(struct ttm_tt *ttm)
|
|
{
|
|
struct address_space *swap_space;
|
|
struct file *swap_storage;
|
|
struct page *from_page;
|
|
struct page *to_page;
|
|
void *from_virtual;
|
|
void *to_virtual;
|
|
int i;
|
|
int ret = -ENOMEM;
|
|
|
|
swap_storage = ttm->swap_storage;
|
|
BUG_ON(swap_storage == NULL);
|
|
|
|
swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
|
|
|
|
for (i = 0; i < ttm->num_pages; ++i) {
|
|
from_page = shmem_read_mapping_page(swap_space, i);
|
|
if (IS_ERR(from_page)) {
|
|
ret = PTR_ERR(from_page);
|
|
goto out_err;
|
|
}
|
|
to_page = ttm->pages[i];
|
|
if (unlikely(to_page == NULL))
|
|
goto out_err;
|
|
|
|
preempt_disable();
|
|
from_virtual = kmap_atomic(from_page, KM_USER0);
|
|
to_virtual = kmap_atomic(to_page, KM_USER1);
|
|
memcpy(to_virtual, from_virtual, PAGE_SIZE);
|
|
kunmap_atomic(to_virtual, KM_USER1);
|
|
kunmap_atomic(from_virtual, KM_USER0);
|
|
preempt_enable();
|
|
page_cache_release(from_page);
|
|
}
|
|
|
|
if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))
|
|
fput(swap_storage);
|
|
ttm->swap_storage = NULL;
|
|
ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
|
|
|
|
return 0;
|
|
out_err:
|
|
return ret;
|
|
}
|
|
|
|
int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)
|
|
{
|
|
struct address_space *swap_space;
|
|
struct file *swap_storage;
|
|
struct page *from_page;
|
|
struct page *to_page;
|
|
void *from_virtual;
|
|
void *to_virtual;
|
|
int i;
|
|
int ret = -ENOMEM;
|
|
|
|
BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
|
|
BUG_ON(ttm->caching_state != tt_cached);
|
|
|
|
if (!persistent_swap_storage) {
|
|
swap_storage = shmem_file_setup("ttm swap",
|
|
ttm->num_pages << PAGE_SHIFT,
|
|
0);
|
|
if (unlikely(IS_ERR(swap_storage))) {
|
|
printk(KERN_ERR "Failed allocating swap storage.\n");
|
|
return PTR_ERR(swap_storage);
|
|
}
|
|
} else
|
|
swap_storage = persistent_swap_storage;
|
|
|
|
swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
|
|
|
|
for (i = 0; i < ttm->num_pages; ++i) {
|
|
from_page = ttm->pages[i];
|
|
if (unlikely(from_page == NULL))
|
|
continue;
|
|
to_page = shmem_read_mapping_page(swap_space, i);
|
|
if (unlikely(IS_ERR(to_page))) {
|
|
ret = PTR_ERR(to_page);
|
|
goto out_err;
|
|
}
|
|
preempt_disable();
|
|
from_virtual = kmap_atomic(from_page, KM_USER0);
|
|
to_virtual = kmap_atomic(to_page, KM_USER1);
|
|
memcpy(to_virtual, from_virtual, PAGE_SIZE);
|
|
kunmap_atomic(to_virtual, KM_USER1);
|
|
kunmap_atomic(from_virtual, KM_USER0);
|
|
preempt_enable();
|
|
set_page_dirty(to_page);
|
|
mark_page_accessed(to_page);
|
|
page_cache_release(to_page);
|
|
}
|
|
|
|
ttm->bdev->driver->ttm_tt_unpopulate(ttm);
|
|
ttm->swap_storage = swap_storage;
|
|
ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
|
|
if (persistent_swap_storage)
|
|
ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;
|
|
|
|
return 0;
|
|
out_err:
|
|
if (!persistent_swap_storage)
|
|
fput(swap_storage);
|
|
|
|
return ret;
|
|
}
|