mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-14 06:46:44 +07:00
cfaed10d1f
When performing a dma_map_sg() call, the number of sg entries to map is required. Using sg_nents to retrieve the number of sg entries will return the total number of entries in the sg list up to the entry marked as the end. If there happen to be unused entries in the list, these will still be counted. Some dma_map_sg() implementations will not handle the unused entries correctly (lib/swiotlb.c) and execute a BUG_ON. The sg_nents_for_len() function will traverse the sg list and return the number of entries required to satisfy the supplied length argument. This can then be supplied to the dma_map_sg() call to successfully map the sg. Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
354 lines
11 KiB
C
354 lines
11 KiB
C
#ifndef _LINUX_SCATTERLIST_H
|
|
#define _LINUX_SCATTERLIST_H
|
|
|
|
#include <linux/string.h>
|
|
#include <linux/bug.h>
|
|
#include <linux/mm.h>
|
|
|
|
#include <asm/types.h>
|
|
#include <asm/scatterlist.h>
|
|
#include <asm/io.h>
|
|
|
|
struct sg_table {
|
|
struct scatterlist *sgl; /* the list */
|
|
unsigned int nents; /* number of mapped entries */
|
|
unsigned int orig_nents; /* original size of list */
|
|
};
|
|
|
|
/*
|
|
* Notes on SG table design.
|
|
*
|
|
* Architectures must provide an unsigned long page_link field in the
|
|
* scatterlist struct. We use that to place the page pointer AND encode
|
|
* information about the sg table as well. The two lower bits are reserved
|
|
* for this information.
|
|
*
|
|
* If bit 0 is set, then the page_link contains a pointer to the next sg
|
|
* table list. Otherwise the next entry is at sg + 1.
|
|
*
|
|
* If bit 1 is set, then this sg entry is the last element in a list.
|
|
*
|
|
* See sg_next().
|
|
*
|
|
*/
|
|
|
|
#define SG_MAGIC 0x87654321
|
|
|
|
/*
|
|
* We overload the LSB of the page pointer to indicate whether it's
|
|
* a valid sg entry, or whether it points to the start of a new scatterlist.
|
|
* Those low bits are there for everyone! (thanks mason :-)
|
|
*/
|
|
#define sg_is_chain(sg) ((sg)->page_link & 0x01)
|
|
#define sg_is_last(sg) ((sg)->page_link & 0x02)
|
|
#define sg_chain_ptr(sg) \
|
|
((struct scatterlist *) ((sg)->page_link & ~0x03))
|
|
|
|
/**
|
|
* sg_assign_page - Assign a given page to an SG entry
|
|
* @sg: SG entry
|
|
* @page: The page
|
|
*
|
|
* Description:
|
|
* Assign page to sg entry. Also see sg_set_page(), the most commonly used
|
|
* variant.
|
|
*
|
|
**/
|
|
static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
|
|
{
|
|
unsigned long page_link = sg->page_link & 0x3;
|
|
|
|
/*
|
|
* In order for the low bit stealing approach to work, pages
|
|
* must be aligned at a 32-bit boundary as a minimum.
|
|
*/
|
|
BUG_ON((unsigned long) page & 0x03);
|
|
#ifdef CONFIG_DEBUG_SG
|
|
BUG_ON(sg->sg_magic != SG_MAGIC);
|
|
BUG_ON(sg_is_chain(sg));
|
|
#endif
|
|
sg->page_link = page_link | (unsigned long) page;
|
|
}
|
|
|
|
/**
|
|
* sg_set_page - Set sg entry to point at given page
|
|
* @sg: SG entry
|
|
* @page: The page
|
|
* @len: Length of data
|
|
* @offset: Offset into page
|
|
*
|
|
* Description:
|
|
* Use this function to set an sg entry pointing at a page, never assign
|
|
* the page directly. We encode sg table information in the lower bits
|
|
* of the page pointer. See sg_page() for looking up the page belonging
|
|
* to an sg entry.
|
|
*
|
|
**/
|
|
static inline void sg_set_page(struct scatterlist *sg, struct page *page,
|
|
unsigned int len, unsigned int offset)
|
|
{
|
|
sg_assign_page(sg, page);
|
|
sg->offset = offset;
|
|
sg->length = len;
|
|
}
|
|
|
|
static inline struct page *sg_page(struct scatterlist *sg)
|
|
{
|
|
#ifdef CONFIG_DEBUG_SG
|
|
BUG_ON(sg->sg_magic != SG_MAGIC);
|
|
BUG_ON(sg_is_chain(sg));
|
|
#endif
|
|
return (struct page *)((sg)->page_link & ~0x3);
|
|
}
|
|
|
|
/**
|
|
* sg_set_buf - Set sg entry to point at given data
|
|
* @sg: SG entry
|
|
* @buf: Data
|
|
* @buflen: Data length
|
|
*
|
|
**/
|
|
static inline void sg_set_buf(struct scatterlist *sg, const void *buf,
|
|
unsigned int buflen)
|
|
{
|
|
#ifdef CONFIG_DEBUG_SG
|
|
BUG_ON(!virt_addr_valid(buf));
|
|
#endif
|
|
sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));
|
|
}
|
|
|
|
/*
|
|
* Loop over each sg element, following the pointer to a new list if necessary
|
|
*/
|
|
#define for_each_sg(sglist, sg, nr, __i) \
|
|
for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))
|
|
|
|
/**
|
|
* sg_chain - Chain two sglists together
|
|
* @prv: First scatterlist
|
|
* @prv_nents: Number of entries in prv
|
|
* @sgl: Second scatterlist
|
|
*
|
|
* Description:
|
|
* Links @prv@ and @sgl@ together, to form a longer scatterlist.
|
|
*
|
|
**/
|
|
static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents,
|
|
struct scatterlist *sgl)
|
|
{
|
|
#ifndef CONFIG_ARCH_HAS_SG_CHAIN
|
|
BUG();
|
|
#endif
|
|
|
|
/*
|
|
* offset and length are unused for chain entry. Clear them.
|
|
*/
|
|
prv[prv_nents - 1].offset = 0;
|
|
prv[prv_nents - 1].length = 0;
|
|
|
|
/*
|
|
* Set lowest bit to indicate a link pointer, and make sure to clear
|
|
* the termination bit if it happens to be set.
|
|
*/
|
|
prv[prv_nents - 1].page_link = ((unsigned long) sgl | 0x01) & ~0x02;
|
|
}
|
|
|
|
/**
|
|
* sg_mark_end - Mark the end of the scatterlist
|
|
* @sg: SG entryScatterlist
|
|
*
|
|
* Description:
|
|
* Marks the passed in sg entry as the termination point for the sg
|
|
* table. A call to sg_next() on this entry will return NULL.
|
|
*
|
|
**/
|
|
static inline void sg_mark_end(struct scatterlist *sg)
|
|
{
|
|
#ifdef CONFIG_DEBUG_SG
|
|
BUG_ON(sg->sg_magic != SG_MAGIC);
|
|
#endif
|
|
/*
|
|
* Set termination bit, clear potential chain bit
|
|
*/
|
|
sg->page_link |= 0x02;
|
|
sg->page_link &= ~0x01;
|
|
}
|
|
|
|
/**
|
|
* sg_unmark_end - Undo setting the end of the scatterlist
|
|
* @sg: SG entryScatterlist
|
|
*
|
|
* Description:
|
|
* Removes the termination marker from the given entry of the scatterlist.
|
|
*
|
|
**/
|
|
static inline void sg_unmark_end(struct scatterlist *sg)
|
|
{
|
|
#ifdef CONFIG_DEBUG_SG
|
|
BUG_ON(sg->sg_magic != SG_MAGIC);
|
|
#endif
|
|
sg->page_link &= ~0x02;
|
|
}
|
|
|
|
/**
|
|
* sg_phys - Return physical address of an sg entry
|
|
* @sg: SG entry
|
|
*
|
|
* Description:
|
|
* This calls page_to_phys() on the page in this sg entry, and adds the
|
|
* sg offset. The caller must know that it is legal to call page_to_phys()
|
|
* on the sg page.
|
|
*
|
|
**/
|
|
static inline dma_addr_t sg_phys(struct scatterlist *sg)
|
|
{
|
|
return page_to_phys(sg_page(sg)) + sg->offset;
|
|
}
|
|
|
|
/**
|
|
* sg_virt - Return virtual address of an sg entry
|
|
* @sg: SG entry
|
|
*
|
|
* Description:
|
|
* This calls page_address() on the page in this sg entry, and adds the
|
|
* sg offset. The caller must know that the sg page has a valid virtual
|
|
* mapping.
|
|
*
|
|
**/
|
|
static inline void *sg_virt(struct scatterlist *sg)
|
|
{
|
|
return page_address(sg_page(sg)) + sg->offset;
|
|
}
|
|
|
|
int sg_nents(struct scatterlist *sg);
|
|
int sg_nents_for_len(struct scatterlist *sg, u64 len);
|
|
struct scatterlist *sg_next(struct scatterlist *);
|
|
struct scatterlist *sg_last(struct scatterlist *s, unsigned int);
|
|
void sg_init_table(struct scatterlist *, unsigned int);
|
|
void sg_init_one(struct scatterlist *, const void *, unsigned int);
|
|
|
|
typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);
|
|
typedef void (sg_free_fn)(struct scatterlist *, unsigned int);
|
|
|
|
void __sg_free_table(struct sg_table *, unsigned int, bool, sg_free_fn *);
|
|
void sg_free_table(struct sg_table *);
|
|
int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int,
|
|
struct scatterlist *, gfp_t, sg_alloc_fn *);
|
|
int sg_alloc_table(struct sg_table *, unsigned int, gfp_t);
|
|
int sg_alloc_table_from_pages(struct sg_table *sgt,
|
|
struct page **pages, unsigned int n_pages,
|
|
unsigned long offset, unsigned long size,
|
|
gfp_t gfp_mask);
|
|
|
|
size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
|
|
void *buf, size_t buflen);
|
|
size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
|
|
void *buf, size_t buflen);
|
|
|
|
size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
|
|
void *buf, size_t buflen, off_t skip);
|
|
size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
|
|
void *buf, size_t buflen, off_t skip);
|
|
|
|
/*
|
|
* Maximum number of entries that will be allocated in one piece, if
|
|
* a list larger than this is required then chaining will be utilized.
|
|
*/
|
|
#define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist))
|
|
|
|
/*
|
|
* sg page iterator
|
|
*
|
|
* Iterates over sg entries page-by-page. On each successful iteration,
|
|
* you can call sg_page_iter_page(@piter) and sg_page_iter_dma_address(@piter)
|
|
* to get the current page and its dma address. @piter->sg will point to the
|
|
* sg holding this page and @piter->sg_pgoffset to the page's page offset
|
|
* within the sg. The iteration will stop either when a maximum number of sg
|
|
* entries was reached or a terminating sg (sg_last(sg) == true) was reached.
|
|
*/
|
|
struct sg_page_iter {
|
|
struct scatterlist *sg; /* sg holding the page */
|
|
unsigned int sg_pgoffset; /* page offset within the sg */
|
|
|
|
/* these are internal states, keep away */
|
|
unsigned int __nents; /* remaining sg entries */
|
|
int __pg_advance; /* nr pages to advance at the
|
|
* next step */
|
|
};
|
|
|
|
bool __sg_page_iter_next(struct sg_page_iter *piter);
|
|
void __sg_page_iter_start(struct sg_page_iter *piter,
|
|
struct scatterlist *sglist, unsigned int nents,
|
|
unsigned long pgoffset);
|
|
/**
|
|
* sg_page_iter_page - get the current page held by the page iterator
|
|
* @piter: page iterator holding the page
|
|
*/
|
|
static inline struct page *sg_page_iter_page(struct sg_page_iter *piter)
|
|
{
|
|
return nth_page(sg_page(piter->sg), piter->sg_pgoffset);
|
|
}
|
|
|
|
/**
|
|
* sg_page_iter_dma_address - get the dma address of the current page held by
|
|
* the page iterator.
|
|
* @piter: page iterator holding the page
|
|
*/
|
|
static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter)
|
|
{
|
|
return sg_dma_address(piter->sg) + (piter->sg_pgoffset << PAGE_SHIFT);
|
|
}
|
|
|
|
/**
|
|
* for_each_sg_page - iterate over the pages of the given sg list
|
|
* @sglist: sglist to iterate over
|
|
* @piter: page iterator to hold current page, sg, sg_pgoffset
|
|
* @nents: maximum number of sg entries to iterate over
|
|
* @pgoffset: starting page offset
|
|
*/
|
|
#define for_each_sg_page(sglist, piter, nents, pgoffset) \
|
|
for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \
|
|
__sg_page_iter_next(piter);)
|
|
|
|
/*
|
|
* Mapping sg iterator
|
|
*
|
|
* Iterates over sg entries mapping page-by-page. On each successful
|
|
* iteration, @miter->page points to the mapped page and
|
|
* @miter->length bytes of data can be accessed at @miter->addr. As
|
|
* long as an interation is enclosed between start and stop, the user
|
|
* is free to choose control structure and when to stop.
|
|
*
|
|
* @miter->consumed is set to @miter->length on each iteration. It
|
|
* can be adjusted if the user can't consume all the bytes in one go.
|
|
* Also, a stopped iteration can be resumed by calling next on it.
|
|
* This is useful when iteration needs to release all resources and
|
|
* continue later (e.g. at the next interrupt).
|
|
*/
|
|
|
|
#define SG_MITER_ATOMIC (1 << 0) /* use kmap_atomic */
|
|
#define SG_MITER_TO_SG (1 << 1) /* flush back to phys on unmap */
|
|
#define SG_MITER_FROM_SG (1 << 2) /* nop */
|
|
|
|
struct sg_mapping_iter {
|
|
/* the following three fields can be accessed directly */
|
|
struct page *page; /* currently mapped page */
|
|
void *addr; /* pointer to the mapped area */
|
|
size_t length; /* length of the mapped area */
|
|
size_t consumed; /* number of consumed bytes */
|
|
struct sg_page_iter piter; /* page iterator */
|
|
|
|
/* these are internal states, keep away */
|
|
unsigned int __offset; /* offset within page */
|
|
unsigned int __remaining; /* remaining bytes on page */
|
|
unsigned int __flags;
|
|
};
|
|
|
|
void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl,
|
|
unsigned int nents, unsigned int flags);
|
|
bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset);
|
|
bool sg_miter_next(struct sg_mapping_iter *miter);
|
|
void sg_miter_stop(struct sg_mapping_iter *miter);
|
|
|
|
#endif /* _LINUX_SCATTERLIST_H */
|