2007-10-22 06:41:49 +07:00
|
|
|
/*
|
2015-09-09 17:40:47 +07:00
|
|
|
* Copyright © 2006-2015, Intel Corporation.
|
|
|
|
*
|
|
|
|
* Authors: Ashok Raj <ashok.raj@intel.com>
|
|
|
|
* Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
|
|
|
|
* David Woodhouse <David.Woodhouse@intel.com>
|
2007-10-22 06:41:49 +07:00
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
|
|
* under the terms and conditions of the GNU General Public License,
|
|
|
|
* version 2, as published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope it will be useful, but WITHOUT
|
|
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
|
* more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License along with
|
|
|
|
* this program; if not, write to the Free Software Foundation, Inc., 59 Temple
|
|
|
|
* Place - Suite 330, Boston, MA 02111-1307 USA.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _INTEL_IOMMU_H_
|
|
|
|
#define _INTEL_IOMMU_H_
|
|
|
|
|
|
|
|
#include <linux/types.h>
|
2008-09-09 22:37:29 +07:00
|
|
|
#include <linux/iova.h>
|
2007-10-22 06:41:49 +07:00
|
|
|
#include <linux/io.h>
|
2015-09-09 17:40:47 +07:00
|
|
|
#include <linux/idr.h>
|
2008-09-09 22:37:29 +07:00
|
|
|
#include <linux/dma_remapping.h>
|
2015-09-09 17:40:47 +07:00
|
|
|
#include <linux/mmu_notifier.h>
|
|
|
|
#include <linux/list.h>
|
2008-07-11 01:16:42 +07:00
|
|
|
#include <asm/cacheflush.h>
|
2008-10-17 08:02:32 +07:00
|
|
|
#include <asm/iommu.h>
|
2008-02-06 16:36:23 +07:00
|
|
|
|
2007-10-22 06:41:49 +07:00
|
|
|
/*
|
|
|
|
* Intel IOMMU register specification per version 1.0 public spec.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define DMAR_VER_REG 0x0 /* Arch version supported by this IOMMU */
|
|
|
|
#define DMAR_CAP_REG 0x8 /* Hardware supported capabilities */
|
|
|
|
#define DMAR_ECAP_REG 0x10 /* Extended capabilities supported */
|
|
|
|
#define DMAR_GCMD_REG 0x18 /* Global command register */
|
|
|
|
#define DMAR_GSTS_REG 0x1c /* Global status register */
|
|
|
|
#define DMAR_RTADDR_REG 0x20 /* Root entry table */
|
|
|
|
#define DMAR_CCMD_REG 0x28 /* Context command reg */
|
|
|
|
#define DMAR_FSTS_REG 0x34 /* Fault Status register */
|
|
|
|
#define DMAR_FECTL_REG 0x38 /* Fault control register */
|
|
|
|
#define DMAR_FEDATA_REG 0x3c /* Fault event interrupt data register */
|
|
|
|
#define DMAR_FEADDR_REG 0x40 /* Fault event interrupt addr register */
|
|
|
|
#define DMAR_FEUADDR_REG 0x44 /* Upper address register */
|
|
|
|
#define DMAR_AFLOG_REG 0x58 /* Advanced Fault control */
|
|
|
|
#define DMAR_PMEN_REG 0x64 /* Enable Protected Memory Region */
|
|
|
|
#define DMAR_PLMBASE_REG 0x68 /* PMRR Low addr */
|
|
|
|
#define DMAR_PLMLIMIT_REG 0x6c /* PMRR low limit */
|
|
|
|
#define DMAR_PHMBASE_REG 0x70 /* pmrr high base addr */
|
|
|
|
#define DMAR_PHMLIMIT_REG 0x78 /* pmrr high limit */
|
2008-07-11 01:16:42 +07:00
|
|
|
#define DMAR_IQH_REG 0x80 /* Invalidation queue head register */
|
|
|
|
#define DMAR_IQT_REG 0x88 /* Invalidation queue tail register */
|
2009-05-18 12:51:35 +07:00
|
|
|
#define DMAR_IQ_SHIFT 4 /* Invalidation queue head/tail shift */
|
2008-07-11 01:16:42 +07:00
|
|
|
#define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */
|
2013-09-13 13:27:32 +07:00
|
|
|
#define DMAR_ICS_REG 0x9c /* Invalidation complete status register */
|
2008-07-11 01:16:43 +07:00
|
|
|
#define DMAR_IRTA_REG 0xb8 /* Interrupt remapping table addr register */
|
2015-10-07 21:37:03 +07:00
|
|
|
#define DMAR_PQH_REG 0xc0 /* Page request queue head register */
|
|
|
|
#define DMAR_PQT_REG 0xc8 /* Page request queue tail register */
|
|
|
|
#define DMAR_PQA_REG 0xd0 /* Page request queue address register */
|
|
|
|
#define DMAR_PRS_REG 0xdc /* Page request status register */
|
|
|
|
#define DMAR_PECTL_REG 0xe0 /* Page request event control register */
|
|
|
|
#define DMAR_PEDATA_REG 0xe4 /* Page request event interrupt data register */
|
|
|
|
#define DMAR_PEADDR_REG 0xe8 /* Page request event interrupt addr register */
|
|
|
|
#define DMAR_PEUADDR_REG 0xec /* Page request event Upper address register */
|
2007-10-22 06:41:49 +07:00
|
|
|
|
|
|
|
#define OFFSET_STRIDE (9)
|
2015-10-14 02:48:21 +07:00
|
|
|
|
|
|
|
#ifdef CONFIG_64BIT
|
|
|
|
#define dmar_readq(a) readq(a)
|
|
|
|
#define dmar_writeq(a,v) writeq(v,a)
|
|
|
|
#else
|
2007-10-29 11:51:16 +07:00
|
|
|
static inline u64 dmar_readq(void __iomem *addr)
|
2007-10-22 06:41:49 +07:00
|
|
|
{
|
|
|
|
u32 lo, hi;
|
|
|
|
lo = readl(addr);
|
|
|
|
hi = readl(addr + 4);
|
|
|
|
return (((u64) hi) << 32) + lo;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void dmar_writeq(void __iomem *addr, u64 val)
|
|
|
|
{
|
|
|
|
writel((u32)val, addr);
|
|
|
|
writel((u32)(val >> 32), addr + 4);
|
|
|
|
}
|
2015-10-14 02:48:21 +07:00
|
|
|
#endif
|
2007-10-22 06:41:49 +07:00
|
|
|
|
|
|
|
#define DMAR_VER_MAJOR(v) (((v) & 0xf0) >> 4)
|
|
|
|
#define DMAR_VER_MINOR(v) ((v) & 0x0f)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Decoding Capability Register
|
|
|
|
*/
|
2015-06-09 12:20:34 +07:00
|
|
|
#define cap_pi_support(c) (((c) >> 59) & 1)
|
2007-10-22 06:41:49 +07:00
|
|
|
#define cap_read_drain(c) (((c) >> 55) & 1)
|
|
|
|
#define cap_write_drain(c) (((c) >> 54) & 1)
|
|
|
|
#define cap_max_amask_val(c) (((c) >> 48) & 0x3f)
|
|
|
|
#define cap_num_fault_regs(c) ((((c) >> 40) & 0xff) + 1)
|
|
|
|
#define cap_pgsel_inv(c) (((c) >> 39) & 1)
|
|
|
|
|
|
|
|
#define cap_super_page_val(c) (((c) >> 34) & 0xf)
|
|
|
|
#define cap_super_offset(c) (((find_first_bit(&cap_super_page_val(c), 4)) \
|
|
|
|
* OFFSET_STRIDE) + 21)
|
|
|
|
|
|
|
|
#define cap_fault_reg_offset(c) ((((c) >> 24) & 0x3ff) * 16)
|
|
|
|
#define cap_max_fault_reg_offset(c) \
|
|
|
|
(cap_fault_reg_offset(c) + cap_num_fault_regs(c) * 16)
|
|
|
|
|
|
|
|
#define cap_zlr(c) (((c) >> 22) & 1)
|
|
|
|
#define cap_isoch(c) (((c) >> 23) & 1)
|
|
|
|
#define cap_mgaw(c) ((((c) >> 16) & 0x3f) + 1)
|
|
|
|
#define cap_sagaw(c) (((c) >> 8) & 0x1f)
|
|
|
|
#define cap_caching_mode(c) (((c) >> 7) & 1)
|
|
|
|
#define cap_phmr(c) (((c) >> 6) & 1)
|
|
|
|
#define cap_plmr(c) (((c) >> 5) & 1)
|
|
|
|
#define cap_rwbf(c) (((c) >> 4) & 1)
|
|
|
|
#define cap_afl(c) (((c) >> 3) & 1)
|
|
|
|
#define cap_ndoms(c) (((unsigned long)1) << (4 + 2 * ((c) & 0x7)))
|
|
|
|
/*
|
|
|
|
* Extended Capability Register
|
|
|
|
*/
|
|
|
|
|
2015-06-09 21:06:55 +07:00
|
|
|
#define ecap_pasid(e) ((e >> 40) & 0x1)
|
2015-03-25 22:43:39 +07:00
|
|
|
#define ecap_pss(e) ((e >> 35) & 0x1f)
|
|
|
|
#define ecap_eafs(e) ((e >> 34) & 0x1)
|
|
|
|
#define ecap_nwfs(e) ((e >> 33) & 0x1)
|
|
|
|
#define ecap_srs(e) ((e >> 31) & 0x1)
|
|
|
|
#define ecap_ers(e) ((e >> 30) & 0x1)
|
|
|
|
#define ecap_prs(e) ((e >> 29) & 0x1)
|
2015-09-09 17:58:59 +07:00
|
|
|
#define ecap_broken_pasid(e) ((e >> 28) & 0x1)
|
2015-03-25 22:43:39 +07:00
|
|
|
#define ecap_dis(e) ((e >> 27) & 0x1)
|
|
|
|
#define ecap_nest(e) ((e >> 26) & 0x1)
|
|
|
|
#define ecap_mts(e) ((e >> 25) & 0x1)
|
|
|
|
#define ecap_ecs(e) ((e >> 24) & 0x1)
|
2007-10-22 06:41:49 +07:00
|
|
|
#define ecap_iotlb_offset(e) ((((e) >> 8) & 0x3ff) * 16)
|
2015-02-13 21:25:24 +07:00
|
|
|
#define ecap_max_iotlb_offset(e) (ecap_iotlb_offset(e) + 16)
|
2007-10-22 06:41:49 +07:00
|
|
|
#define ecap_coherent(e) ((e) & 0x1)
|
2008-07-11 01:16:42 +07:00
|
|
|
#define ecap_qis(e) ((e) & 0x2)
|
2009-04-25 07:30:20 +07:00
|
|
|
#define ecap_pass_through(e) ((e >> 6) & 0x1)
|
2008-07-11 01:16:40 +07:00
|
|
|
#define ecap_eim_support(e) ((e >> 4) & 0x1)
|
|
|
|
#define ecap_ir_support(e) ((e >> 3) & 0x1)
|
2009-05-18 12:51:37 +07:00
|
|
|
#define ecap_dev_iotlb_support(e) (((e) >> 2) & 0x1)
|
2008-07-11 01:16:44 +07:00
|
|
|
#define ecap_max_handle_mask(e) ((e >> 20) & 0xf)
|
2009-03-18 14:33:05 +07:00
|
|
|
#define ecap_sc_support(e) ((e >> 7) & 0x1) /* Snooping Control */
|
2007-10-22 06:41:49 +07:00
|
|
|
|
|
|
|
/* IOTLB_REG */
|
2008-10-17 06:31:55 +07:00
|
|
|
#define DMA_TLB_FLUSH_GRANU_OFFSET 60
|
2007-10-22 06:41:49 +07:00
|
|
|
#define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60)
|
|
|
|
#define DMA_TLB_DSI_FLUSH (((u64)2) << 60)
|
|
|
|
#define DMA_TLB_PSI_FLUSH (((u64)3) << 60)
|
|
|
|
#define DMA_TLB_IIRG(type) ((type >> 60) & 7)
|
|
|
|
#define DMA_TLB_IAIG(val) (((val) >> 57) & 7)
|
|
|
|
#define DMA_TLB_READ_DRAIN (((u64)1) << 49)
|
|
|
|
#define DMA_TLB_WRITE_DRAIN (((u64)1) << 48)
|
|
|
|
#define DMA_TLB_DID(id) (((u64)((id) & 0xffff)) << 32)
|
|
|
|
#define DMA_TLB_IVT (((u64)1) << 63)
|
|
|
|
#define DMA_TLB_IH_NONLEAF (((u64)1) << 6)
|
|
|
|
#define DMA_TLB_MAX_SIZE (0x3f)
|
|
|
|
|
2008-07-11 01:16:42 +07:00
|
|
|
/* INVALID_DESC */
|
2008-10-17 06:31:55 +07:00
|
|
|
#define DMA_CCMD_INVL_GRANU_OFFSET 61
|
2008-07-11 01:16:42 +07:00
|
|
|
#define DMA_ID_TLB_GLOBAL_FLUSH (((u64)1) << 3)
|
|
|
|
#define DMA_ID_TLB_DSI_FLUSH (((u64)2) << 3)
|
|
|
|
#define DMA_ID_TLB_PSI_FLUSH (((u64)3) << 3)
|
|
|
|
#define DMA_ID_TLB_READ_DRAIN (((u64)1) << 7)
|
|
|
|
#define DMA_ID_TLB_WRITE_DRAIN (((u64)1) << 6)
|
|
|
|
#define DMA_ID_TLB_DID(id) (((u64)((id & 0xffff) << 16)))
|
|
|
|
#define DMA_ID_TLB_IH_NONLEAF (((u64)1) << 6)
|
|
|
|
#define DMA_ID_TLB_ADDR(addr) (addr)
|
|
|
|
#define DMA_ID_TLB_ADDR_MASK(mask) (mask)
|
|
|
|
|
2008-02-08 19:18:38 +07:00
|
|
|
/* PMEN_REG */
|
|
|
|
#define DMA_PMEN_EPM (((u32)1)<<31)
|
|
|
|
#define DMA_PMEN_PRS (((u32)1)<<0)
|
|
|
|
|
2007-10-22 06:41:49 +07:00
|
|
|
/* GCMD_REG */
|
|
|
|
#define DMA_GCMD_TE (((u32)1) << 31)
|
|
|
|
#define DMA_GCMD_SRTP (((u32)1) << 30)
|
|
|
|
#define DMA_GCMD_SFL (((u32)1) << 29)
|
|
|
|
#define DMA_GCMD_EAFL (((u32)1) << 28)
|
|
|
|
#define DMA_GCMD_WBF (((u32)1) << 27)
|
2008-07-11 01:16:43 +07:00
|
|
|
#define DMA_GCMD_QIE (((u32)1) << 26)
|
|
|
|
#define DMA_GCMD_SIRTP (((u32)1) << 24)
|
|
|
|
#define DMA_GCMD_IRE (((u32) 1) << 25)
|
2009-04-03 16:15:47 +07:00
|
|
|
#define DMA_GCMD_CFI (((u32) 1) << 23)
|
2007-10-22 06:41:49 +07:00
|
|
|
|
|
|
|
/* GSTS_REG */
|
|
|
|
#define DMA_GSTS_TES (((u32)1) << 31)
|
|
|
|
#define DMA_GSTS_RTPS (((u32)1) << 30)
|
|
|
|
#define DMA_GSTS_FLS (((u32)1) << 29)
|
|
|
|
#define DMA_GSTS_AFLS (((u32)1) << 28)
|
|
|
|
#define DMA_GSTS_WBFS (((u32)1) << 27)
|
2008-07-11 01:16:43 +07:00
|
|
|
#define DMA_GSTS_QIES (((u32)1) << 26)
|
|
|
|
#define DMA_GSTS_IRTPS (((u32)1) << 24)
|
|
|
|
#define DMA_GSTS_IRES (((u32)1) << 25)
|
2009-04-03 16:15:47 +07:00
|
|
|
#define DMA_GSTS_CFIS (((u32)1) << 23)
|
2007-10-22 06:41:49 +07:00
|
|
|
|
2015-03-25 22:43:39 +07:00
|
|
|
/* DMA_RTADDR_REG */
|
|
|
|
#define DMA_RTADDR_RTT (((u64)1) << 11)
|
|
|
|
|
2007-10-22 06:41:49 +07:00
|
|
|
/* CCMD_REG */
|
|
|
|
#define DMA_CCMD_ICC (((u64)1) << 63)
|
|
|
|
#define DMA_CCMD_GLOBAL_INVL (((u64)1) << 61)
|
|
|
|
#define DMA_CCMD_DOMAIN_INVL (((u64)2) << 61)
|
|
|
|
#define DMA_CCMD_DEVICE_INVL (((u64)3) << 61)
|
|
|
|
#define DMA_CCMD_FM(m) (((u64)((m) & 0x3)) << 32)
|
|
|
|
#define DMA_CCMD_MASK_NOBIT 0
|
|
|
|
#define DMA_CCMD_MASK_1BIT 1
|
|
|
|
#define DMA_CCMD_MASK_2BIT 2
|
|
|
|
#define DMA_CCMD_MASK_3BIT 3
|
|
|
|
#define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16)
|
|
|
|
#define DMA_CCMD_DID(d) ((u64)((d) & 0xffff))
|
|
|
|
|
|
|
|
/* FECTL_REG */
|
|
|
|
#define DMA_FECTL_IM (((u32)1) << 31)
|
|
|
|
|
|
|
|
/* FSTS_REG */
|
|
|
|
#define DMA_FSTS_PPF ((u32)2)
|
|
|
|
#define DMA_FSTS_PFO ((u32)1)
|
2009-01-04 15:28:52 +07:00
|
|
|
#define DMA_FSTS_IQE (1 << 4)
|
2009-05-18 12:51:35 +07:00
|
|
|
#define DMA_FSTS_ICE (1 << 5)
|
|
|
|
#define DMA_FSTS_ITE (1 << 6)
|
2007-10-22 06:41:49 +07:00
|
|
|
#define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff)
|
|
|
|
|
|
|
|
/* FRCD_REG, 32 bits access */
|
|
|
|
#define DMA_FRCD_F (((u32)1) << 31)
|
|
|
|
#define dma_frcd_type(d) ((d >> 30) & 1)
|
|
|
|
#define dma_frcd_fault_reason(c) (c & 0xff)
|
|
|
|
#define dma_frcd_source_id(c) (c & 0xffff)
|
2008-10-17 08:02:32 +07:00
|
|
|
/* low 64 bit */
|
|
|
|
#define dma_frcd_page_addr(d) (d & (((u64)-1) << PAGE_SHIFT))
|
|
|
|
|
2016-02-15 19:42:38 +07:00
|
|
|
/* PRS_REG */
|
|
|
|
#define DMA_PRS_PPR ((u32)1)
|
|
|
|
|
2008-10-17 08:02:32 +07:00
|
|
|
#define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \
|
|
|
|
do { \
|
|
|
|
cycles_t start_time = get_cycles(); \
|
|
|
|
while (1) { \
|
|
|
|
sts = op(iommu->reg + offset); \
|
|
|
|
if (cond) \
|
|
|
|
break; \
|
2008-07-11 01:16:41 +07:00
|
|
|
if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start_time))\
|
2008-10-17 08:02:32 +07:00
|
|
|
panic("DMAR hardware is malfunctioning\n"); \
|
|
|
|
cpu_relax(); \
|
|
|
|
} \
|
|
|
|
} while (0)
|
2008-07-11 01:16:41 +07:00
|
|
|
|
2008-07-11 01:16:42 +07:00
|
|
|
#define QI_LENGTH 256 /* queue length */
|
|
|
|
|
|
|
|
enum {
|
|
|
|
QI_FREE,
|
|
|
|
QI_IN_USE,
|
2009-05-18 12:51:35 +07:00
|
|
|
QI_DONE,
|
|
|
|
QI_ABORT
|
2008-07-11 01:16:42 +07:00
|
|
|
};
|
|
|
|
|
|
|
|
#define QI_CC_TYPE 0x1
|
|
|
|
#define QI_IOTLB_TYPE 0x2
|
|
|
|
#define QI_DIOTLB_TYPE 0x3
|
|
|
|
#define QI_IEC_TYPE 0x4
|
|
|
|
#define QI_IWD_TYPE 0x5
|
2015-09-09 17:40:47 +07:00
|
|
|
#define QI_EIOTLB_TYPE 0x6
|
|
|
|
#define QI_PC_TYPE 0x7
|
|
|
|
#define QI_DEIOTLB_TYPE 0x8
|
2015-10-08 05:35:18 +07:00
|
|
|
#define QI_PGRP_RESP_TYPE 0x9
|
|
|
|
#define QI_PSTRM_RESP_TYPE 0xa
|
2008-07-11 01:16:42 +07:00
|
|
|
|
|
|
|
#define QI_IEC_SELECTIVE (((u64)1) << 4)
|
|
|
|
#define QI_IEC_IIDEX(idx) (((u64)(idx & 0xffff) << 32))
|
|
|
|
#define QI_IEC_IM(m) (((u64)(m & 0x1f) << 27))
|
|
|
|
|
|
|
|
#define QI_IWD_STATUS_DATA(d) (((u64)d) << 32)
|
|
|
|
#define QI_IWD_STATUS_WRITE (((u64)1) << 5)
|
|
|
|
|
2008-10-17 06:31:55 +07:00
|
|
|
#define QI_IOTLB_DID(did) (((u64)did) << 16)
|
|
|
|
#define QI_IOTLB_DR(dr) (((u64)dr) << 7)
|
|
|
|
#define QI_IOTLB_DW(dw) (((u64)dw) << 6)
|
|
|
|
#define QI_IOTLB_GRAN(gran) (((u64)gran) >> (DMA_TLB_FLUSH_GRANU_OFFSET-4))
|
2008-10-17 08:02:32 +07:00
|
|
|
#define QI_IOTLB_ADDR(addr) (((u64)addr) & VTD_PAGE_MASK)
|
2008-10-17 06:31:55 +07:00
|
|
|
#define QI_IOTLB_IH(ih) (((u64)ih) << 6)
|
|
|
|
#define QI_IOTLB_AM(am) (((u8)am))
|
|
|
|
|
|
|
|
#define QI_CC_FM(fm) (((u64)fm) << 48)
|
|
|
|
#define QI_CC_SID(sid) (((u64)sid) << 32)
|
|
|
|
#define QI_CC_DID(did) (((u64)did) << 16)
|
|
|
|
#define QI_CC_GRAN(gran) (((u64)gran) >> (DMA_CCMD_INVL_GRANU_OFFSET-4))
|
|
|
|
|
2009-05-18 12:51:35 +07:00
|
|
|
#define QI_DEV_IOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32)
|
|
|
|
#define QI_DEV_IOTLB_QDEP(qdep) (((qdep) & 0x1f) << 16)
|
|
|
|
#define QI_DEV_IOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK)
|
|
|
|
#define QI_DEV_IOTLB_SIZE 1
|
|
|
|
#define QI_DEV_IOTLB_MAX_INVS 32
|
|
|
|
|
2015-09-09 17:40:47 +07:00
|
|
|
#define QI_PC_PASID(pasid) (((u64)pasid) << 32)
|
|
|
|
#define QI_PC_DID(did) (((u64)did) << 16)
|
|
|
|
#define QI_PC_GRAN(gran) (((u64)gran) << 4)
|
|
|
|
|
|
|
|
#define QI_PC_ALL_PASIDS (QI_PC_TYPE | QI_PC_GRAN(0))
|
|
|
|
#define QI_PC_PASID_SEL (QI_PC_TYPE | QI_PC_GRAN(1))
|
|
|
|
|
|
|
|
#define QI_EIOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK)
|
|
|
|
#define QI_EIOTLB_GL(gl) (((u64)gl) << 7)
|
|
|
|
#define QI_EIOTLB_IH(ih) (((u64)ih) << 6)
|
|
|
|
#define QI_EIOTLB_AM(am) (((u64)am))
|
|
|
|
#define QI_EIOTLB_PASID(pasid) (((u64)pasid) << 32)
|
|
|
|
#define QI_EIOTLB_DID(did) (((u64)did) << 16)
|
|
|
|
#define QI_EIOTLB_GRAN(gran) (((u64)gran) << 4)
|
|
|
|
|
|
|
|
#define QI_DEV_EIOTLB_ADDR(a) ((u64)(a) & VTD_PAGE_MASK)
|
|
|
|
#define QI_DEV_EIOTLB_SIZE (((u64)1) << 11)
|
|
|
|
#define QI_DEV_EIOTLB_GLOB(g) ((u64)g)
|
|
|
|
#define QI_DEV_EIOTLB_PASID(p) (((u64)p) << 32)
|
|
|
|
#define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32)
|
|
|
|
#define QI_DEV_EIOTLB_QDEP(qd) (((qd) & 0x1f) << 16)
|
|
|
|
#define QI_DEV_EIOTLB_MAX_INVS 32
|
|
|
|
|
2015-10-08 05:35:18 +07:00
|
|
|
#define QI_PGRP_IDX(idx) (((u64)(idx)) << 55)
|
|
|
|
#define QI_PGRP_PRIV(priv) (((u64)(priv)) << 32)
|
|
|
|
#define QI_PGRP_RESP_CODE(res) ((u64)(res))
|
|
|
|
#define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32)
|
|
|
|
#define QI_PGRP_DID(did) (((u64)(did)) << 16)
|
|
|
|
#define QI_PGRP_PASID_P(p) (((u64)(p)) << 4)
|
|
|
|
|
|
|
|
#define QI_PSTRM_ADDR(addr) (((u64)(addr)) & VTD_PAGE_MASK)
|
|
|
|
#define QI_PSTRM_DEVFN(devfn) (((u64)(devfn)) << 4)
|
|
|
|
#define QI_PSTRM_RESP_CODE(res) ((u64)(res))
|
|
|
|
#define QI_PSTRM_IDX(idx) (((u64)(idx)) << 55)
|
|
|
|
#define QI_PSTRM_PRIV(priv) (((u64)(priv)) << 32)
|
|
|
|
#define QI_PSTRM_BUS(bus) (((u64)(bus)) << 24)
|
|
|
|
#define QI_PSTRM_PASID(pasid) (((u64)(pasid)) << 4)
|
|
|
|
|
|
|
|
#define QI_RESP_SUCCESS 0x0
|
|
|
|
#define QI_RESP_INVALID 0x1
|
|
|
|
#define QI_RESP_FAILURE 0xf
|
|
|
|
|
2015-09-09 17:40:47 +07:00
|
|
|
#define QI_GRAN_ALL_ALL 0
|
|
|
|
#define QI_GRAN_NONG_ALL 1
|
|
|
|
#define QI_GRAN_NONG_PASID 2
|
|
|
|
#define QI_GRAN_PSI_PASID 3
|
|
|
|
|
2008-07-11 01:16:42 +07:00
|
|
|
struct qi_desc {
|
|
|
|
u64 low, high;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct q_inval {
|
2011-07-19 22:02:07 +07:00
|
|
|
raw_spinlock_t q_lock;
|
2008-07-11 01:16:42 +07:00
|
|
|
struct qi_desc *desc; /* invalidation queue */
|
|
|
|
int *desc_status; /* desc status */
|
|
|
|
int free_head; /* first free entry */
|
|
|
|
int free_tail; /* last free entry */
|
|
|
|
int free_cnt;
|
|
|
|
};
|
|
|
|
|
2011-08-24 07:05:25 +07:00
|
|
|
#ifdef CONFIG_IRQ_REMAP
|
2008-07-11 01:16:43 +07:00
|
|
|
/* 1MB - maximum possible interrupt remapping table size */
|
|
|
|
#define INTR_REMAP_PAGE_ORDER 8
|
|
|
|
#define INTR_REMAP_TABLE_REG_SIZE 0xf
|
2015-06-12 20:00:21 +07:00
|
|
|
#define INTR_REMAP_TABLE_REG_SIZE_MASK 0xf
|
2008-07-11 01:16:43 +07:00
|
|
|
|
2008-07-11 01:16:44 +07:00
|
|
|
#define INTR_REMAP_TABLE_ENTRIES 65536
|
|
|
|
|
2015-04-13 13:11:32 +07:00
|
|
|
struct irq_domain;
|
|
|
|
|
2008-07-11 01:16:43 +07:00
|
|
|
struct ir_table {
|
|
|
|
struct irte *base;
|
2014-01-06 13:18:08 +07:00
|
|
|
unsigned long *bitmap;
|
2008-07-11 01:16:43 +07:00
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
2008-10-17 06:31:56 +07:00
|
|
|
struct iommu_flush {
|
2009-05-10 23:16:06 +07:00
|
|
|
void (*flush_context)(struct intel_iommu *iommu, u16 did, u16 sid,
|
|
|
|
u8 fm, u64 type);
|
2009-05-11 01:58:49 +07:00
|
|
|
void (*flush_iotlb)(struct intel_iommu *iommu, u16 did, u64 addr,
|
|
|
|
unsigned int size_order, u64 type);
|
2008-10-17 06:31:56 +07:00
|
|
|
};
|
|
|
|
|
2009-03-28 04:22:42 +07:00
|
|
|
enum {
|
|
|
|
SR_DMAR_FECTL_REG,
|
|
|
|
SR_DMAR_FEDATA_REG,
|
|
|
|
SR_DMAR_FEADDR_REG,
|
|
|
|
SR_DMAR_FEUADDR_REG,
|
|
|
|
MAX_SR_DMAR_REGS
|
|
|
|
};
|
|
|
|
|
2015-06-12 15:14:02 +07:00
|
|
|
#define VTD_FLAG_TRANS_PRE_ENABLED (1 << 0)
|
|
|
|
#define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1)
|
|
|
|
|
2015-03-24 21:54:56 +07:00
|
|
|
struct pasid_entry;
|
|
|
|
struct pasid_state_entry;
|
2015-10-08 05:35:18 +07:00
|
|
|
struct page_req_dsc;
|
2015-03-24 21:54:56 +07:00
|
|
|
|
2007-10-22 06:41:49 +07:00
|
|
|
struct intel_iommu {
|
|
|
|
void __iomem *reg; /* Pointer to hardware regs, virtual addr */
|
2012-06-05 04:29:02 +07:00
|
|
|
u64 reg_phys; /* physical address of hw register set */
|
|
|
|
u64 reg_size; /* size of hw register set */
|
2007-10-22 06:41:49 +07:00
|
|
|
u64 cap;
|
|
|
|
u64 ecap;
|
|
|
|
u32 gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */
|
2011-07-19 21:19:51 +07:00
|
|
|
raw_spinlock_t register_lock; /* protect register handling */
|
2008-07-11 01:16:36 +07:00
|
|
|
int seq_id; /* sequence id of the iommu */
|
2008-12-08 14:34:06 +07:00
|
|
|
int agaw; /* agaw of this iommu */
|
2009-04-25 07:30:20 +07:00
|
|
|
int msagaw; /* max sagaw of this iommu */
|
2015-10-07 21:37:03 +07:00
|
|
|
unsigned int irq, pr_irq;
|
2014-03-10 03:49:45 +07:00
|
|
|
u16 segment; /* PCI segment# */
|
2009-03-17 07:04:55 +07:00
|
|
|
unsigned char name[13]; /* Device Name */
|
2008-07-11 01:16:35 +07:00
|
|
|
|
2011-08-24 07:05:25 +07:00
|
|
|
#ifdef CONFIG_INTEL_IOMMU
|
2008-07-11 01:16:35 +07:00
|
|
|
unsigned long *domain_ids; /* bitmap of domains */
|
2015-07-21 15:41:21 +07:00
|
|
|
struct dmar_domain ***domains; /* ptr to domains */
|
2008-07-11 01:16:35 +07:00
|
|
|
spinlock_t lock; /* protect context, domain ids */
|
2007-10-22 06:41:49 +07:00
|
|
|
struct root_entry *root_entry; /* virtual address */
|
|
|
|
|
2008-10-17 06:31:56 +07:00
|
|
|
struct iommu_flush flush;
|
2015-03-24 21:54:56 +07:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_INTEL_IOMMU_SVM
|
|
|
|
/* These are large and need to be contiguous, so we allocate just
|
|
|
|
* one for now. We'll maybe want to rethink that if we truly give
|
|
|
|
* devices away to userspace processes (e.g. for DPDK) and don't
|
|
|
|
* want to trust that userspace will use *only* the PASID it was
|
|
|
|
* told to. But while it's all driver-arbitrated, we're fine. */
|
|
|
|
struct pasid_entry *pasid_table;
|
|
|
|
struct pasid_state_entry *pasid_state_table;
|
2015-10-08 05:35:18 +07:00
|
|
|
struct page_req_dsc *prq;
|
|
|
|
unsigned char prq_name[16]; /* Name for PRQ interrupt */
|
2015-09-09 17:40:47 +07:00
|
|
|
struct idr pasid_idr;
|
2016-09-12 09:49:11 +07:00
|
|
|
u32 pasid_max;
|
2008-07-11 01:16:35 +07:00
|
|
|
#endif
|
2008-07-11 01:16:42 +07:00
|
|
|
struct q_inval *qi; /* Queued invalidation info */
|
2009-03-28 04:22:42 +07:00
|
|
|
u32 *iommu_state; /* Store iommu states between suspend and resume.*/
|
|
|
|
|
2011-08-24 07:05:25 +07:00
|
|
|
#ifdef CONFIG_IRQ_REMAP
|
2008-07-11 01:16:43 +07:00
|
|
|
struct ir_table *ir_table; /* Interrupt remapping info */
|
2015-04-13 13:11:32 +07:00
|
|
|
struct irq_domain *ir_domain;
|
|
|
|
struct irq_domain *ir_msi_domain;
|
2008-07-11 01:16:43 +07:00
|
|
|
#endif
|
iommu/vt-d: Make use of IOMMU sysfs support
Register our DRHD IOMMUs, cross link devices, and provide a base set
of attributes for the IOMMU. Note that IRQ remapping support parses
the DMAR table very early in boot, well before the iommu_class can
reasonably be setup, so our registration is split between
intel_iommu_init(), which occurs later, and alloc_iommu(), which
typically occurs much earlier, but may happen at any time later
with IOMMU hot-add support.
On a typical desktop system, this provides the following (pruned):
$ find /sys | grep dmar
/sys/devices/virtual/iommu/dmar0
/sys/devices/virtual/iommu/dmar0/devices
/sys/devices/virtual/iommu/dmar0/devices/0000:00:02.0
/sys/devices/virtual/iommu/dmar0/intel-iommu
/sys/devices/virtual/iommu/dmar0/intel-iommu/cap
/sys/devices/virtual/iommu/dmar0/intel-iommu/ecap
/sys/devices/virtual/iommu/dmar0/intel-iommu/address
/sys/devices/virtual/iommu/dmar0/intel-iommu/version
/sys/devices/virtual/iommu/dmar1
/sys/devices/virtual/iommu/dmar1/devices
/sys/devices/virtual/iommu/dmar1/devices/0000:00:00.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:01.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:16.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1a.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1b.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1c.0
...
/sys/devices/virtual/iommu/dmar1/intel-iommu
/sys/devices/virtual/iommu/dmar1/intel-iommu/cap
/sys/devices/virtual/iommu/dmar1/intel-iommu/ecap
/sys/devices/virtual/iommu/dmar1/intel-iommu/address
/sys/devices/virtual/iommu/dmar1/intel-iommu/version
/sys/class/iommu/dmar0
/sys/class/iommu/dmar1
(devices also link back to the dmar units)
This makes address, version, capabilities, and extended capabilities
available, just like printed on boot. I've tried not to duplicate
data that can be found in the DMAR table, with the exception of the
address, which provides an easy way to associate the sysfs device with
a DRHD entry in the DMAR. It's tempting to add scopes and RMRR data
here, but the full DMAR table is already exposed under /sys/firmware/
and therefore already provides a way for userspace to learn such
details.
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Joerg Roedel <jroedel@suse.de>
2014-06-13 05:12:31 +07:00
|
|
|
struct device *iommu_dev; /* IOMMU-sysfs device */
|
2009-10-03 01:01:21 +07:00
|
|
|
int node;
|
2015-06-12 15:14:02 +07:00
|
|
|
u32 flags; /* Software defined flags */
|
2007-10-22 06:41:49 +07:00
|
|
|
};
|
|
|
|
|
2008-07-11 01:16:42 +07:00
|
|
|
static inline void __iommu_flush_cache(
|
|
|
|
struct intel_iommu *iommu, void *addr, int size)
|
|
|
|
{
|
|
|
|
if (!ecap_coherent(iommu->ecap))
|
|
|
|
clflush_cache_range(addr, size);
|
|
|
|
}
|
|
|
|
|
2008-07-11 01:16:35 +07:00
|
|
|
extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev);
|
2009-05-18 12:51:34 +07:00
|
|
|
extern int dmar_find_matched_atsr_unit(struct pci_dev *dev);
|
2008-07-11 01:16:35 +07:00
|
|
|
|
2008-07-11 01:16:43 +07:00
|
|
|
extern int dmar_enable_qi(struct intel_iommu *iommu);
|
2009-03-17 07:04:56 +07:00
|
|
|
extern void dmar_disable_qi(struct intel_iommu *iommu);
|
2009-03-28 04:22:42 +07:00
|
|
|
extern int dmar_reenable_qi(struct intel_iommu *iommu);
|
2008-07-11 01:16:43 +07:00
|
|
|
extern void qi_global_iec(struct intel_iommu *iommu);
|
2007-10-22 06:41:55 +07:00
|
|
|
|
2009-05-10 23:16:06 +07:00
|
|
|
extern void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid,
|
|
|
|
u8 fm, u64 type);
|
2009-05-11 01:58:49 +07:00
|
|
|
extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
|
|
|
|
unsigned int size_order, u64 type);
|
2009-05-18 12:51:35 +07:00
|
|
|
extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
|
|
|
|
u64 addr, unsigned mask);
|
2008-10-17 06:31:55 +07:00
|
|
|
|
2009-01-04 15:28:52 +07:00
|
|
|
extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu);
|
2008-09-09 22:37:29 +07:00
|
|
|
|
2009-09-09 23:05:39 +07:00
|
|
|
extern int dmar_ir_support(void);
|
|
|
|
|
2015-09-09 17:40:47 +07:00
|
|
|
#ifdef CONFIG_INTEL_IOMMU_SVM
|
2015-03-24 21:54:56 +07:00
|
|
|
extern int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu);
|
|
|
|
extern int intel_svm_free_pasid_tables(struct intel_iommu *iommu);
|
2015-10-08 05:35:18 +07:00
|
|
|
extern int intel_svm_enable_prq(struct intel_iommu *iommu);
|
|
|
|
extern int intel_svm_finish_prq(struct intel_iommu *iommu);
|
2015-03-24 21:54:56 +07:00
|
|
|
|
2015-10-13 23:18:10 +07:00
|
|
|
struct svm_dev_ops;
|
|
|
|
|
2015-09-09 17:40:47 +07:00
|
|
|
struct intel_svm_dev {
|
|
|
|
struct list_head list;
|
|
|
|
struct rcu_head rcu;
|
|
|
|
struct device *dev;
|
2015-10-13 23:18:10 +07:00
|
|
|
struct svm_dev_ops *ops;
|
2015-09-09 17:40:47 +07:00
|
|
|
int users;
|
|
|
|
u16 did;
|
|
|
|
u16 dev_iotlb:1;
|
|
|
|
u16 sid, qdep;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct intel_svm {
|
|
|
|
struct mmu_notifier notifier;
|
|
|
|
struct mm_struct *mm;
|
|
|
|
struct intel_iommu *iommu;
|
2015-10-15 19:59:14 +07:00
|
|
|
int flags;
|
2015-09-09 17:40:47 +07:00
|
|
|
int pasid;
|
|
|
|
struct list_head devs;
|
|
|
|
};
|
|
|
|
|
|
|
|
extern int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev);
|
|
|
|
extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev);
|
|
|
|
#endif
|
|
|
|
|
iommu/vt-d: Make use of IOMMU sysfs support
Register our DRHD IOMMUs, cross link devices, and provide a base set
of attributes for the IOMMU. Note that IRQ remapping support parses
the DMAR table very early in boot, well before the iommu_class can
reasonably be setup, so our registration is split between
intel_iommu_init(), which occurs later, and alloc_iommu(), which
typically occurs much earlier, but may happen at any time later
with IOMMU hot-add support.
On a typical desktop system, this provides the following (pruned):
$ find /sys | grep dmar
/sys/devices/virtual/iommu/dmar0
/sys/devices/virtual/iommu/dmar0/devices
/sys/devices/virtual/iommu/dmar0/devices/0000:00:02.0
/sys/devices/virtual/iommu/dmar0/intel-iommu
/sys/devices/virtual/iommu/dmar0/intel-iommu/cap
/sys/devices/virtual/iommu/dmar0/intel-iommu/ecap
/sys/devices/virtual/iommu/dmar0/intel-iommu/address
/sys/devices/virtual/iommu/dmar0/intel-iommu/version
/sys/devices/virtual/iommu/dmar1
/sys/devices/virtual/iommu/dmar1/devices
/sys/devices/virtual/iommu/dmar1/devices/0000:00:00.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:01.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:16.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1a.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1b.0
/sys/devices/virtual/iommu/dmar1/devices/0000:00:1c.0
...
/sys/devices/virtual/iommu/dmar1/intel-iommu
/sys/devices/virtual/iommu/dmar1/intel-iommu/cap
/sys/devices/virtual/iommu/dmar1/intel-iommu/ecap
/sys/devices/virtual/iommu/dmar1/intel-iommu/address
/sys/devices/virtual/iommu/dmar1/intel-iommu/version
/sys/class/iommu/dmar0
/sys/class/iommu/dmar1
(devices also link back to the dmar units)
This makes address, version, capabilities, and extended capabilities
available, just like printed on boot. I've tried not to duplicate
data that can be found in the DMAR table, with the exception of the
address, which provides an easy way to associate the sysfs device with
a DRHD entry in the DMAR. It's tempting to add scopes and RMRR data
here, but the full DMAR table is already exposed under /sys/firmware/
and therefore already provides a way for userspace to learn such
details.
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Joerg Roedel <jroedel@suse.de>
2014-06-13 05:12:31 +07:00
|
|
|
extern const struct attribute_group *intel_iommu_groups[];
|
|
|
|
|
2007-10-22 06:41:49 +07:00
|
|
|
#endif
|