AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-12-04 02:06:43 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

powerpc updates for 5.1

Browse Source 
Notable changes:
 
  - Enable THREAD_INFO_IN_TASK to move thread_info off the stack.
 
  - A big series from Christoph reworking our DMA code to use more of the generic
    infrastructure, as he said:
    "This series switches the powerpc port to use the generic swiotlb and
     noncoherent dma ops, and to use more generic code for the coherent direct
     mapping, as well as removing a lot of dead code."
 
  - Increase our vmalloc space to 512T with the Hash MMU on modern CPUs, allowing
    us to support machines with larger amounts of total RAM or distance between
    nodes.
 
  - Two series from Christophe, one to optimise TLB miss handlers on 6xx, and
    another to optimise the way STRICT_KERNEL_RWX is implemented on some 32-bit
    CPUs.
 
  - Support for KCOV coverage instrumentation which means we can run syzkaller
    and discover even more bugs in our code.
 
 And as always many clean-ups, reworks and minor fixes etc.
 
 Thanks to:
  Alan Modra, Alexey Kardashevskiy, Alistair Popple, Andrea Arcangeli, Andrew
  Donnellan, Aneesh Kumar K.V, Aravinda Prasad, Balbir Singh, Brajeswar Ghosh,
  Breno Leitao, Christian Lamparter, Christian Zigotzky, Christophe Leroy,
  Christoph Hellwig, Corentin Labbe, Daniel Axtens, David Gibson, Diana Craciun,
  Firoz Khan, Gustavo A. R. Silva, Igor Stoppa, Joe Lawrence, Joel Stanley,
  Jonathan Neuschäfer, Jordan Niethe, Laurent Dufour, Madhavan Srinivasan, Mahesh
  Salgaonkar, Mark Cave-Ayland, Masahiro Yamada, Mathieu Malaterre, Matteo Croce,
  Meelis Roos, Michael W. Bringmann, Nathan Chancellor, Nathan Fontenot, Nicholas
  Piggin, Nick Desaulniers, Nicolai Stange, Oliver O'Halloran, Paul Mackerras,
  Peter Xu, PrasannaKumar Muralidharan, Qian Cai, Rashmica Gupta, Reza Arbab,
  Robert P. J. Day, Russell Currey, Sabyasachi Gupta, Sam Bobroff, Sandipan Das,
  Sergey Senozhatsky, Souptick Joarder, Stewart Smith, Tyrel Datwyler, Vaibhav
  Jain, YueHaibing.
 -----BEGIN PGP SIGNATURE-----
 
 iQIcBAABAgAGBQJcgRJlAAoJEFHr6jzI4aWAL9oP+gPlrZgyaAg/51lmubLtlbtk
 QuGU8EiuJZoJD1OHrMPtppBOY7rQZOxJe58AoPig8wTvs+j/TxJ25fmiZncnf5U2
 PC8QAjbj0UmQHgy+K30sUeOnDg9tdkHKHJ5/ecjJcvykkqsjyMnV7biFQ1cOA0HT
 LflXHEEtiG9P9u7jZoAhtnfpgn1/l9mhTYMe26J1fqvC0164qMDFaXDTQXyDfyvG
 gmuqccGMawSk7IdagmQxwXtwyfwOnarmGn+n31XKRejApGZ/pjiEA23JOJOaJcia
 m76Jy3roao6sEtCUNpBFXEtwOy9POy3OiGy6yg/9896tDMvG84OuO6ltV1nFGawL
 PmwE+ug63L4g/HWxZyAeb26T2oTTp/YIaKQPtsq4d286pvg/qr2KPNzFoAEhmJqU
 yLrebv276pVeiLpLmCLPvcPj9t76vWKZaUm0FoE+zUDg7Rl7Alow8A/c4tdjOI6y
 QwpbCiYseyiJ32lCZZdbN7Cy6+iM6vb3i1oNKc8MVqhBGTwLJnTU0ruPBSvCaRvD
 NoQWO1RWpNu/BuivuLEKS9q3AoxenGwiqowxGhdVmI3Oc9jGWcEYlduR00VDYPVp
 /RCfwtTY5NyC++h5cnbz8aLJ1hBXG5m79CXfprV+zPWeiLPCaMT6w9Y5QUS2wqA+
 EZ734NknDJOjaHc4cGdZ
 =Z9bb
 -----END PGP SIGNATURE-----

Merge tag 'powerpc-5.1-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux

Pull powerpc updates from Michael Ellerman:
 "Notable changes:

   - Enable THREAD_INFO_IN_TASK to move thread_info off the stack.

   - A big series from Christoph reworking our DMA code to use more of
     the generic infrastructure, as he said:
       "This series switches the powerpc port to use the generic swiotlb
        and noncoherent dma ops, and to use more generic code for the
        coherent direct mapping, as well as removing a lot of dead
        code."

   - Increase our vmalloc space to 512T with the Hash MMU on modern
     CPUs, allowing us to support machines with larger amounts of total
     RAM or distance between nodes.

   - Two series from Christophe, one to optimise TLB miss handlers on
     6xx, and another to optimise the way STRICT_KERNEL_RWX is
     implemented on some 32-bit CPUs.

   - Support for KCOV coverage instrumentation which means we can run
     syzkaller and discover even more bugs in our code.

  And as always many clean-ups, reworks and minor fixes etc.

  Thanks to: Alan Modra, Alexey Kardashevskiy, Alistair Popple, Andrea
  Arcangeli, Andrew Donnellan, Aneesh Kumar K.V, Aravinda Prasad, Balbir
  Singh, Brajeswar Ghosh, Breno Leitao, Christian Lamparter, Christian
  Zigotzky, Christophe Leroy, Christoph Hellwig, Corentin Labbe, Daniel
  Axtens, David Gibson, Diana Craciun, Firoz Khan, Gustavo A. R. Silva,
  Igor Stoppa, Joe Lawrence, Joel Stanley, Jonathan Neuschäfer, Jordan
  Niethe, Laurent Dufour, Madhavan Srinivasan, Mahesh Salgaonkar, Mark
  Cave-Ayland, Masahiro Yamada, Mathieu Malaterre, Matteo Croce, Meelis
  Roos, Michael W. Bringmann, Nathan Chancellor, Nathan Fontenot,
  Nicholas Piggin, Nick Desaulniers, Nicolai Stange, Oliver O'Halloran,
  Paul Mackerras, Peter Xu, PrasannaKumar Muralidharan, Qian Cai,
  Rashmica Gupta, Reza Arbab, Robert P. J. Day, Russell Currey,
  Sabyasachi Gupta, Sam Bobroff, Sandipan Das, Sergey Senozhatsky,
  Souptick Joarder, Stewart Smith, Tyrel Datwyler, Vaibhav Jain,
  YueHaibing"

* tag 'powerpc-5.1-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux: (200 commits)
  powerpc/32: Clear on-stack exception marker upon exception return
  powerpc: Remove export of save_stack_trace_tsk_reliable()
  powerpc/mm: fix "section_base" set but not used
  powerpc/mm: Fix "sz" set but not used warning
  powerpc/mm: Check secondary hash page table
  powerpc: remove nargs from __SYSCALL
  powerpc/64s: Fix unrelocated interrupt trampoline address test
  powerpc/powernv/ioda: Fix locked_vm counting for memory used by IOMMU tables
  powerpc/fsl: Fix the flush of branch predictor.
  powerpc/powernv: Make opal log only readable by root
  powerpc/xmon: Fix opcode being uninitialized in print_insn_powerpc
  powerpc/powernv: move OPAL call wrapper tracing and interrupt handling to C
  powerpc/64s: Fix data interrupts vs d-side MCE reentrancy
  powerpc/64s: Prepare to handle data interrupts vs d-side MCE reentrancy
  powerpc/64s: system reset interrupt preserve HSRRs
  powerpc/64s: Fix HV NMI vs HV interrupt recoverability test
  powerpc/mm/hash: Handle mmap_min_addr correctly in get_unmapped_area topdown search
  powerpc/hugetlb: Handle mmap_min_addr correctly in get_unmapped_area callback
  selftests/powerpc: Remove duplicate header
  powerpc sstep: Add support for modsd, modud instructions
  ...
This commit is contained in:
Linus Torvalds 2019-03-07 12:56:26 -08:00
commit 6c3ac11343
216 changed files with 3626 additions and 2998 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
arch/powerpc/Kconfig
View File

@ -119,9 +119,6 @@ config GENERIC_HWEIGHT
bool
default y
config ARCH_HAS_DMA_SET_COHERENT_MASK
bool
config PPC
bool
default y
@ -131,10 +128,10 @@ config PPC
select ARCH_32BIT_OFF_T if PPC32
select ARCH_HAS_DEBUG_VIRTUAL
select ARCH_HAS_DEVMEM_IS_ALLOWED
select ARCH_HAS_DMA_SET_COHERENT_MASK
select ARCH_HAS_ELF_RANDOMIZE
select ARCH_HAS_FORTIFY_SOURCE
select ARCH_HAS_GCOV_PROFILE_ALL
select ARCH_HAS_KCOV
select ARCH_HAS_PHYS_TO_DMA
select ARCH_HAS_PMEM_API if PPC64
select ARCH_HAS_PTE_SPECIAL
@ -203,7 +200,7 @@ config PPC
select HAVE_IOREMAP_PROT
select HAVE_IRQ_EXIT_ON_IRQ_STACK
select HAVE_KERNEL_GZIP
select HAVE_KERNEL_XZ if PPC_BOOK3S
select HAVE_KERNEL_XZ if PPC_BOOK3S || 44x
select HAVE_KPROBES
select HAVE_KPROBES_ON_FTRACE
select HAVE_KRETPROBES
@ -222,7 +219,7 @@ config PPC
select HAVE_PERF_USER_STACK_DUMP
select HAVE_RCU_TABLE_FREE if SMP
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RELIABLE_STACKTRACE if PPC64 && CPU_LITTLE_ENDIAN
select HAVE_RELIABLE_STACKTRACE if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN
select HAVE_SYSCALL_TRACEPOINTS
select HAVE_VIRT_CPU_ACCOUNTING
select HAVE_IRQ_TIME_ACCOUNTING
@ -243,6 +240,7 @@ config PPC
select RTC_LIB
select SPARSE_IRQ
select SYSCTL_EXCEPTION_TRACE
select THREAD_INFO_IN_TASK
select VIRT_TO_BUS if !PPC64
#
# Please keep this list sorted alphabetically.
@ -253,9 +251,6 @@ config PPC_BARRIER_NOSPEC
default y
depends on PPC_BOOK3S_64 || PPC_FSL_BOOK3E
config GENERIC_CSUM
def_bool n
config EARLY_PRINTK
bool
default y
@ -475,9 +470,6 @@ config ARCH_CPU_PROBE_RELEASE
config ARCH_ENABLE_MEMORY_HOTPLUG
def_bool y
config ARCH_HAS_WALK_MEMORY
def_bool y
config ARCH_ENABLE_MEMORY_HOTREMOVE
def_bool y
@ -693,7 +685,7 @@ config PPC_16K_PAGES
config PPC_64K_PAGES
bool "64k page size"
depends on !PPC_FSL_BOOK3E && (44x || PPC_BOOK3S_64 || PPC_BOOK3E_64)
depends on 44x || PPC_BOOK3S_64
select HAVE_ARCH_SOFT_DIRTY if PPC_BOOK3S_64
config PPC_256K_PAGES
@ -711,6 +703,13 @@ config PPC_256K_PAGES
endchoice
config PPC_PAGE_SHIFT
int
default 18 if PPC_256K_PAGES
default 16 if PPC_64K_PAGES
default 14 if PPC_16K_PAGES
default 12
config THREAD_SHIFT
int "Thread shift" if EXPERT
range 13 15
@ -721,6 +720,59 @@ config THREAD_SHIFT
Used to define the stack size. The default is almost always what you
want. Only change this if you know what you are doing.
config ETEXT_SHIFT_BOOL
bool "Set custom etext alignment" if STRICT_KERNEL_RWX && \
(PPC_BOOK3S_32 || PPC_8xx)
depends on ADVANCED_OPTIONS
help
This option allows you to set the kernel end of text alignment. When
RAM is mapped by blocks, the alignment needs to fit the size and
number of possible blocks. The default should be OK for most configs.
Say N here unless you know what you are doing.
config ETEXT_SHIFT
int "_etext shift" if ETEXT_SHIFT_BOOL
range 17 28 if STRICT_KERNEL_RWX && PPC_BOOK3S_32
range 19 23 if STRICT_KERNEL_RWX && PPC_8xx
default 17 if STRICT_KERNEL_RWX && PPC_BOOK3S_32
default 19 if STRICT_KERNEL_RWX && PPC_8xx
default PPC_PAGE_SHIFT
help
On Book3S 32 (603+), IBATs are used to map kernel text.
Smaller is the alignment, greater is the number of necessary IBATs.
On 8xx, large pages (512kb or 8M) are used to map kernel linear
memory. Aligning to 8M reduces TLB misses as only 8M pages are used
in that case.
config DATA_SHIFT_BOOL
bool "Set custom data alignment" if STRICT_KERNEL_RWX && \
(PPC_BOOK3S_32 || PPC_8xx)
depends on ADVANCED_OPTIONS
help
This option allows you to set the kernel data alignment. When
RAM is mapped by blocks, the alignment needs to fit the size and
number of possible blocks. The default should be OK for most configs.
Say N here unless you know what you are doing.
config DATA_SHIFT
int "Data shift" if DATA_SHIFT_BOOL
default 24 if STRICT_KERNEL_RWX && PPC64
range 17 28 if STRICT_KERNEL_RWX && PPC_BOOK3S_32
range 19 23 if STRICT_KERNEL_RWX && PPC_8xx
default 22 if STRICT_KERNEL_RWX && PPC_BOOK3S_32
default 23 if STRICT_KERNEL_RWX && PPC_8xx
default PPC_PAGE_SHIFT
help
On Book3S 32 (603+), DBATs are used to map kernel text and rodata RO.
Smaller is the alignment, greater is the number of necessary DBATs.
On 8xx, large pages (512kb or 8M) are used to map kernel linear
memory. Aligning to 8M reduces TLB misses as only 8M pages are used
in that case.
config FORCE_MAX_ZONEORDER
int "Maximum zone order"
range 8 9 if PPC64 && PPC_64K_PAGES
@ -887,6 +939,7 @@ config FSL_SOC
config FSL_PCI
bool
select ARCH_HAS_DMA_SET_MASK
select PPC_INDIRECT_PCI
select PCI_QUIRKS

4
arch/powerpc/Kconfig.debug
View File

@ -361,10 +361,6 @@ config PPC_PTDUMP
If you are unsure, say N.
config PPC_HTDUMP
def_bool y
depends on PPC_PTDUMP && PPC_BOOK3S_64
config PPC_FAST_ENDIAN_SWITCH
bool "Deprecated fast endian-switch syscall"
depends on DEBUG_KERNEL && PPC_BOOK3S_64

11
arch/powerpc/Makefile
View File

@ -213,9 +213,9 @@ endif
asinstr := $(call as-instr,lis 9$(comma)foo@high,-DHAVE_AS_ATHIGH=1)
KBUILD_CPPFLAGS += -Iarch/$(ARCH) $(asinstr)
KBUILD_AFLAGS += -Iarch/$(ARCH) $(AFLAGS-y)
KBUILD_AFLAGS += $(AFLAGS-y)
KBUILD_CFLAGS += $(call cc-option,-msoft-float)
KBUILD_CFLAGS += -pipe -Iarch/$(ARCH) $(CFLAGS-y)
KBUILD_CFLAGS += -pipe $(CFLAGS-y)
CPP = $(CC) -E $(KBUILD_CFLAGS)
CHECKFLAGS += -m$(BITS) -D__powerpc__ -D__powerpc$(BITS)__
@ -427,6 +427,13 @@ else
endif
endif
ifdef CONFIG_SMP
prepare: task_cpu_prepare
task_cpu_prepare: prepare0
$(eval KBUILD_CFLAGS += -D_TASK_CPU=$(shell awk '{if ($$2 == "TASK_CPU") print $$3;}' include/generated/asm-offsets.h))
endif
# Check toolchain versions:
# - gcc-4.6 is the minimum kernel-wide version so nothing required.
checkbin:

1
arch/powerpc/boot/dts/Makefile
View File

@ -4,3 +4,4 @@ subdir-y += fsl
dtstree := $(srctree)/$(src)
dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts))
dtb-$(CONFIG_XILINX_VIRTEX440_GENERIC_BOARD) += virtex440-ml507.dtb virtex440-ml510.dtb

2
arch/powerpc/boot/dts/akebono.dts
View File

@ -40,7 +40,7 @@ cpu@0 {
d-cache-size = <32768>;
dcr-controller;
dcr-access-method = "native";
status = "ok";
status = "okay";
};
cpu@1 {
device_type = "cpu";

2
arch/powerpc/boot/dts/bluestone.dts
View File

@ -109,7 +109,7 @@ UIC3: interrupt-controller3 {
OCM: ocm@400040000 {
compatible = "ibm,ocm";
status = "ok";
status = "okay";
cell-index = <1>;
/* configured in U-Boot */
reg = <4 0x00040000 0x8000>; /* 32K */

2
arch/powerpc/boot/dts/currituck.dts
View File

@ -39,7 +39,7 @@ cpu@0 {
d-cache-size = <32768>;
dcr-controller;
dcr-access-method = "native";
status = "ok";
status = "okay";
};
cpu@1 {
device_type = "cpu";

2
arch/powerpc/boot/dts/iss4xx-mpic.dts
View File

@ -43,7 +43,7 @@ cpu@0 {
d-cache-size = <32768>;
dcr-controller;
dcr-access-method = "native";
status = "ok";
status = "okay";
};
cpu@1 {
device_type = "cpu";

22
arch/powerpc/boot/dts/wii.dts
View File

@ -14,6 +14,7 @@
/dts-v1/;
#include <dt-bindings/gpio/gpio.h>
#include <dt-bindings/input/input.h>
/*
* This is commented-out for now.
@ -187,6 +188,11 @@ GPIO: gpio@d8000c0 {
"DEBUG0", "DEBUG1", "DEBUG2", "DEBUG3",
"DEBUG4", "DEBUG5", "DEBUG6", "DEBUG7";
interrupt-controller;
#interrupt-cells = <2>;
interrupts = <10>;
interrupt-parent = <&PIC1>;
/*
* This is commented out while a standard binding
* for i2c over gpio is defined.
@ -235,5 +241,21 @@ drive-slot {
panic-indicator;
};
};
gpio-keys {
compatible = "gpio-keys";
power {
label = "Power Button";
gpios = <&GPIO 0 GPIO_ACTIVE_HIGH>;
linux,code = <KEY_POWER>;
};
eject {
label = "Eject Button";
gpios = <&GPIO 6 GPIO_ACTIVE_HIGH>;
linux,code = <KEY_EJECTCD>;
};
};
};

14
arch/powerpc/include/asm/asm-prototypes.h
View File

@ -23,8 +23,8 @@
#include <uapi/asm/ucontext.h>
/* SMP */
extern struct thread_info *current_set[NR_CPUS];
extern struct thread_info *secondary_ti;
extern struct task_struct *current_set[NR_CPUS];
extern struct task_struct *secondary_current;
void start_secondary(void *unused);
/* kexec */
@ -37,13 +37,11 @@ void kexec_copy_flush(struct kimage *image);
extern struct static_key hcall_tracepoint_key;
void __trace_hcall_entry(unsigned long opcode, unsigned long *args);
void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf);
/* OPAL tracing */
#ifdef CONFIG_JUMP_LABEL
extern struct static_key opal_tracepoint_key;
#endif
void __trace_opal_entry(unsigned long opcode, unsigned long *args);
void __trace_opal_exit(long opcode, unsigned long retval);
/* OPAL */
int64_t __opal_call(int64_t a0, int64_t a1, int64_t a2, int64_t a3,
int64_t a4, int64_t a5, int64_t a6, int64_t a7,
int64_t opcode, uint64_t msr);
/* VMX copying */
int enter_vmx_usercopy(void);

2
arch/powerpc/include/asm/book3s/32/mmu-hash.h
View File

@ -92,6 +92,8 @@ typedef struct {
unsigned long vdso_base;
} mm_context_t;
void update_bats(void);
/* patch sites */
extern s32 patch__hash_page_A0, patch__hash_page_A1, patch__hash_page_A2;
extern s32 patch__hash_page_B, patch__hash_page_C;

11
arch/powerpc/include/asm/book3s/32/pgtable.h
View File

@ -174,7 +174,18 @@ static inline bool pte_user(pte_t pte)
* of RAM. -- Cort
*/
#define VMALLOC_OFFSET (0x1000000) /* 16M */
/*
* With CONFIG_STRICT_KERNEL_RWX, kernel segments are set NX. But when modules
* are used, NX cannot be set on VMALLOC space. So vmalloc VM space and linear
* memory shall not share segments.
*/
#if defined(CONFIG_STRICT_KERNEL_RWX) && defined(CONFIG_MODULES)
#define VMALLOC_START ((_ALIGN((long)high_memory, 256L << 20) + VMALLOC_OFFSET) & \
~(VMALLOC_OFFSET - 1))
#else
#define VMALLOC_START ((((long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)))
#endif
#define VMALLOC_END ioremap_bot
#ifndef __ASSEMBLY__

32
arch/powerpc/include/asm/book3s/64/hash.h
View File

@ -40,22 +40,36 @@
#else
#define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE)
#endif
/*
* Define the address range of the kernel non-linear virtual area
* Define the address range of the kernel non-linear virtual area. In contrast
* to the linear mapping, this is managed using the kernel page tables and then
* inserted into the hash page table to actually take effect, similarly to user
* mappings.
*/
#define H_KERN_VIRT_START ASM_CONST(0xD000000000000000)
#define H_KERN_VIRT_SIZE ASM_CONST(0x0000400000000000) /* 64T */
/*
* The vmalloc space starts at the beginning of that region, and
* occupies half of it on hash CPUs and a quarter of it on Book3E
* (we keep a quarter for the virtual memmap)
* Allow virtual mapping of one context size.
* 512TB for 64K page size
* 64TB for 4K page size
*/
#define H_VMALLOC_START H_KERN_VIRT_START
#define H_VMALLOC_SIZE ASM_CONST(0x380000000000) /* 56T */
#define H_VMALLOC_END (H_VMALLOC_START + H_VMALLOC_SIZE)
#define H_KERN_VIRT_SIZE (1UL << MAX_EA_BITS_PER_CONTEXT)
#define H_KERN_IO_START H_VMALLOC_END
/*
* 8TB IO mapping size
*/
#define H_KERN_IO_SIZE ASM_CONST(0x80000000000) /* 8T */
/*
* The vmalloc space starts at the beginning of the kernel non-linear virtual
* region, and occupies 504T (64K) or 56T (4K)
*/
#define H_VMALLOC_START H_KERN_VIRT_START
#define H_VMALLOC_SIZE (H_KERN_VIRT_SIZE - H_KERN_IO_SIZE)
#define H_VMALLOC_END (H_VMALLOC_START + H_VMALLOC_SIZE)
#define H_KERN_IO_START H_VMALLOC_END
/*
* Region IDs

2
arch/powerpc/include/asm/book3s/64/mmu-hash.h
View File

@ -23,7 +23,7 @@
*/
#include <asm/book3s/64/pgtable.h>
#include <asm/bug.h>
#include <asm/processor.h>
#include <asm/task_size_64.h>
#include <asm/cpu_has_feature.h>
/*

8
arch/powerpc/include/asm/book3s/64/pgalloc.h
View File

@ -111,7 +111,7 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
{
pgd_set(pgd, __pgtable_ptr_val(pud) | PGD_VAL_BITS);
*pgd = __pgd(__pgtable_ptr_val(pud) | PGD_VAL_BITS);
}
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
@ -138,7 +138,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
pud_set(pud, __pgtable_ptr_val(pmd) | PUD_VAL_BITS);
*pud = __pud(__pgtable_ptr_val(pmd) | PUD_VAL_BITS);
}
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
@ -176,13 +176,13 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
pte_t *pte)
{
pmd_set(pmd, __pgtable_ptr_val(pte) | PMD_VAL_BITS);
*pmd = __pmd(__pgtable_ptr_val(pte) | PMD_VAL_BITS);
}
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
pgtable_t pte_page)
{
pmd_set(pmd, __pgtable_ptr_val(pte_page) | PMD_VAL_BITS);
*pmd = __pmd(__pgtable_ptr_val(pte_page) | PMD_VAL_BITS);
}
static inline pgtable_t pmd_pgtable(pmd_t pmd)

16
arch/powerpc/include/asm/book3s/64/pgtable.h
View File

@ -811,7 +811,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
return hash__set_pte_at(mm, addr, ptep, pte, percpu);
}
#define _PAGE_CACHE_CTL (_PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT)
#define _PAGE_CACHE_CTL (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT)
#define pgprot_noncached pgprot_noncached
static inline pgprot_t pgprot_noncached(pgprot_t prot)
@ -851,11 +851,6 @@ static inline bool pte_ci(pte_t pte)
return false;
}
static inline void pmd_set(pmd_t *pmdp, unsigned long val)
{
*pmdp = __pmd(val);
}
static inline void pmd_clear(pmd_t *pmdp)
{
*pmdp = __pmd(0);
@ -887,11 +882,6 @@ static inline int pmd_bad(pmd_t pmd)
return hash__pmd_bad(pmd);
}
static inline void pud_set(pud_t *pudp, unsigned long val)
{
*pudp = __pud(val);
}
static inline void pud_clear(pud_t *pudp)
{
*pudp = __pud(0);
@ -934,10 +924,6 @@ static inline bool pud_access_permitted(pud_t pud, bool write)
}
#define pgd_write(pgd) pte_write(pgd_pte(pgd))
static inline void pgd_set(pgd_t *pgdp, unsigned long val)
{
*pgdp = __pgd(val);
}
static inline void pgd_clear(pgd_t *pgdp)
{

30
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
View File

@ -13,8 +13,32 @@ static inline int mmu_get_ap(int psize)
#ifdef CONFIG_PPC_RADIX_MMU
extern void radix__tlbiel_all(unsigned int action);
extern void radix__flush_tlb_lpid_page(unsigned int lpid,
unsigned long addr,
unsigned long page_size);
extern void radix__flush_pwc_lpid(unsigned int lpid);
extern void radix__flush_tlb_lpid(unsigned int lpid);
extern void radix__local_flush_tlb_lpid_guest(unsigned int lpid);
#else
static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); };
static inline void radix__flush_tlb_lpid_page(unsigned int lpid,
unsigned long addr,
unsigned long page_size)
{
WARN_ON(1);
}
static inline void radix__flush_pwc_lpid(unsigned int lpid)
{
WARN_ON(1);
}
static inline void radix__flush_tlb_lpid(unsigned int lpid)
{
WARN_ON(1);
}
static inline void radix__local_flush_tlb_lpid_guest(unsigned int lpid)
{
WARN_ON(1);
}
#endif
extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
@ -49,12 +73,6 @@ extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
extern void radix__flush_tlb_all(void);
extern void radix__flush_tlb_lpid_page(unsigned int lpid,
unsigned long addr,
unsigned long page_size);
extern void radix__flush_pwc_lpid(unsigned int lpid);
extern void radix__flush_tlb_lpid(unsigned int lpid);
extern void radix__local_flush_tlb_lpid(unsigned int lpid);
extern void radix__local_flush_tlb_lpid_guest(unsigned int lpid);
#endif

4
arch/powerpc/include/asm/checksum.h
View File

@ -9,9 +9,6 @@
* 2 of the License, or (at your option) any later version.
*/
#ifdef CONFIG_GENERIC_CSUM
#include <asm-generic/checksum.h>
#else
#include <linux/bitops.h>
#include <linux/in6.h>
/*
@ -217,6 +214,5 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
const struct in6_addr *daddr,
__u32 len, __u8 proto, __wsum sum);
#endif
#endif /* __KERNEL__ */
#endif

10
arch/powerpc/include/asm/device.h
View File

@ -19,6 +19,11 @@ struct iommu_table;
* drivers/macintosh/macio_asic.c
*/
struct dev_archdata {
/*
* Set to %true if the dma_iommu_ops are requested to use a direct
* window instead of dynamically mapping memory.
*/
bool iommu_bypass : 1;
/*
* These two used to be a union. However, with the hybrid ops we need
* both so here we store both a DMA offset for direct mappings and
@ -33,9 +38,6 @@ struct dev_archdata {
#ifdef CONFIG_IOMMU_API
void *iommu_domain;
#endif
#ifdef CONFIG_SWIOTLB
dma_addr_t max_direct_dma_addr;
#endif
#ifdef CONFIG_PPC64
struct pci_dn *pci_data;
#endif
@ -54,6 +56,4 @@ struct pdev_archdata {
u64 dma_mask;
};
#define ARCH_HAS_DMA_GET_REQUIRED_MASK
#endif /* _ASM_POWERPC_DEVICE_H */

18
arch/powerpc/include/asm/dma-direct.h
View File

@ -4,26 +4,24 @@
static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
{
#ifdef CONFIG_SWIOTLB
struct dev_archdata *sd = &dev->archdata;
if (sd->max_direct_dma_addr && addr + size > sd->max_direct_dma_addr)
return false;
#endif
if (!dev->dma_mask)
return false;
return addr + size - 1 <= *dev->dma_mask;
return addr + size - 1 <=
min_not_zero(*dev->dma_mask, dev->bus_dma_mask);
}
static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{
return paddr + get_dma_offset(dev);
if (!dev)
return paddr + PCI_DRAM_OFFSET;
return paddr + dev->archdata.dma_offset;
}
static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
{
return daddr - get_dma_offset(dev);
if (!dev)
return daddr - PCI_DRAM_OFFSET;
return daddr - dev->archdata.dma_offset;
}
#endif /* ASM_POWERPC_DMA_DIRECT_H */

92
arch/powerpc/include/asm/dma-mapping.h
View File

@ -1,74 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2004 IBM
*
* Implements the generic device dma API for powerpc.
* the pci and vio busses
*/
#ifndef _ASM_DMA_MAPPING_H
#define _ASM_DMA_MAPPING_H
#ifdef __KERNEL__
#include <linux/types.h>
#include <linux/cache.h>
/* need struct page definitions */
#include <linux/mm.h>
#include <linux/scatterlist.h>
#include <linux/dma-debug.h>
#include <asm/io.h>
#include <asm/swiotlb.h>
/* Some dma direct funcs must be visible for use in other dma_ops */
extern void *__dma_nommu_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
unsigned long attrs);
extern void __dma_nommu_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs);
extern int dma_nommu_mmap_coherent(struct device *dev,
struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t handle,
size_t size, unsigned long attrs);
#ifdef CONFIG_NOT_COHERENT_CACHE
/*
* DMA-consistent mapping functions for PowerPCs that don't support
* cache snooping. These allocate/free a region of uncached mapped
* memory space for use with DMA devices. Alternatively, you could
* allocate the space "normally" and use the cache management functions
* to ensure it is consistent.
*/
struct device;
extern void __dma_sync(void *vaddr, size_t size, int direction);
extern void __dma_sync_page(struct page *page, unsigned long offset,
size_t size, int direction);
extern unsigned long __dma_get_coherent_pfn(unsigned long cpu_addr);
#else /* ! CONFIG_NOT_COHERENT_CACHE */
/*
* Cache coherent cores.
*/
#define __dma_sync(addr, size, rw) ((void)0)
#define __dma_sync_page(pg, off, sz, rw) ((void)0)
#endif /* ! CONFIG_NOT_COHERENT_CACHE */
static inline unsigned long device_to_mask(struct device *dev)
{
if (dev->dma_mask && *dev->dma_mask)
return *dev->dma_mask;
/* Assume devices without mask can take 32 bit addresses */
return 0xfffffffful;
}
/*
* Available generic sets of operations
*/
#ifdef CONFIG_PPC64
extern struct dma_map_ops dma_iommu_ops;
#endif
extern const struct dma_map_ops dma_nommu_ops;
static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
{
@ -80,31 +15,4 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
return NULL;
}
/*
* get_dma_offset()
*
* Get the dma offset on configurations where the dma address can be determined
* from the physical address by looking at a simple offset. Direct dma and
* swiotlb use this function, but it is typically not used by implementations
* with an iommu.
*/
static inline dma_addr_t get_dma_offset(struct device *dev)
{
if (dev)
return dev->archdata.dma_offset;
return PCI_DRAM_OFFSET;
}
static inline void set_dma_offset(struct device *dev, dma_addr_t off)
{
if (dev)
dev->archdata.dma_offset = off;
}
#define HAVE_ARCH_DMA_SET_MASK 1
extern u64 __dma_get_required_mask(struct device *dev);
#endif /* __KERNEL__ */
#endif /* _ASM_DMA_MAPPING_H */

10
arch/powerpc/include/asm/eeh.h
View File

@ -219,7 +219,8 @@ struct eeh_ops {
};
extern int eeh_subsystem_flags;
extern int eeh_max_freezes;
extern u32 eeh_max_freezes;
extern bool eeh_debugfs_no_recover;
extern struct eeh_ops *eeh_ops;
extern raw_spinlock_t confirm_error_lock;
@ -293,14 +294,14 @@ void eeh_add_device_late(struct pci_dev *);
void eeh_add_device_tree_late(struct pci_bus *);
void eeh_add_sysfs_files(struct pci_bus *);
void eeh_remove_device(struct pci_dev *);
int eeh_unfreeze_pe(struct eeh_pe *pe, bool sw_state);
int eeh_unfreeze_pe(struct eeh_pe *pe);
int eeh_pe_reset_and_recover(struct eeh_pe *pe);
int eeh_dev_open(struct pci_dev *pdev);
void eeh_dev_release(struct pci_dev *pdev);
struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group);
int eeh_pe_set_option(struct eeh_pe *pe, int option);
int eeh_pe_get_state(struct eeh_pe *pe);
int eeh_pe_reset(struct eeh_pe *pe, int option);
int eeh_pe_reset(struct eeh_pe *pe, int option, bool include_passed);
int eeh_pe_configure(struct eeh_pe *pe);
int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func,
unsigned long addr, unsigned long mask);
@ -460,6 +461,9 @@ static inline void eeh_readsl(const volatile void __iomem *addr, void * buf,
eeh_check_failure(addr);
}
void eeh_cache_debugfs_init(void);
#endif /* CONFIG_PPC64 */
#endif /* __KERNEL__ */
#endif /* _POWERPC_EEH_H */

1
arch/powerpc/include/asm/eeh_event.h
View File

@ -33,6 +33,7 @@ struct eeh_event {
int eeh_event_init(void);
int eeh_send_failure_event(struct eeh_pe *pe);
int __eeh_send_failure_event(struct eeh_pe *pe);
void eeh_remove_event(struct eeh_pe *pe, bool force);
void eeh_handle_normal_event(struct eeh_pe *pe);
void eeh_handle_special_event(void);

4
arch/powerpc/include/asm/exception-64s.h
View File

@ -671,7 +671,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
#define RUNLATCH_ON \
BEGIN_FTR_SECTION \
CURRENT_THREAD_INFO(r3, r1); \
ld r3, PACA_THREAD_INFO(r13); \
ld r4,TI_LOCAL_FLAGS(r3); \
andi. r0,r4,_TLF_RUNLATCH; \
beql ppc64_runlatch_on_trampoline; \
@ -721,7 +721,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
#ifdef CONFIG_PPC_970_NAP
#define FINISH_NAP \
BEGIN_FTR_SECTION \
CURRENT_THREAD_INFO(r11, r1); \
ld r11, PACA_THREAD_INFO(r13); \
ld r9,TI_LOCAL_FLAGS(r11); \
andi. r10,r9,_TLF_NAPPING; \
bnel power4_fixup_nap; \

2
arch/powerpc/include/asm/hvsi.h
View File

@ -64,7 +64,7 @@ struct hvsi_priv {
unsigned int inbuf_len; /* data in input buffer */
unsigned char inbuf[HVSI_INBUF_SIZE];
unsigned int inbuf_cur; /* Cursor in input buffer */
unsigned int inbuf_pktlen; /* packet lenght from cursor */
unsigned int inbuf_pktlen; /* packet length from cursor */
atomic_t seqno; /* packet sequence number */
unsigned int opened:1; /* driver opened */
unsigned int established:1; /* protocol established */

17
arch/powerpc/include/asm/iommu.h
View File

@ -237,6 +237,7 @@ static inline void iommu_del_device(struct device *dev)
}
#endif /* !CONFIG_IOMMU_API */
u64 dma_iommu_get_required_mask(struct device *dev);
#else
static inline void *get_iommu_table_base(struct device *dev)
@ -318,5 +319,21 @@ extern void iommu_release_ownership(struct iommu_table *tbl);
extern enum dma_data_direction iommu_tce_direction(unsigned long tce);
extern unsigned long iommu_direction_to_tce_perm(enum dma_data_direction dir);
#ifdef CONFIG_PPC_CELL_NATIVE
extern bool iommu_fixed_is_weak;
#else
#define iommu_fixed_is_weak false
#endif
extern const struct dma_map_ops dma_iommu_ops;
static inline unsigned long device_to_mask(struct device *dev)
{
if (dev->dma_mask && *dev->dma_mask)
return *dev->dma_mask;
/* Assume devices without mask can take 32 bit addresses */
return 0xfffffffful;
}
#endif /* __KERNEL__ */
#endif /* _ASM_IOMMU_H */

3
arch/powerpc/include/asm/ipic.h
View File

@ -69,10 +69,7 @@ enum ipic_mcp_irq {
IPIC_MCP_MU = 7,
};
extern void ipic_set_highest_priority(unsigned int irq);
extern void ipic_set_default_priority(void);
extern void ipic_enable_mcp(enum ipic_mcp_irq mcp_irq);
extern void ipic_disable_mcp(enum ipic_mcp_irq mcp_irq);
extern u32 ipic_get_mcp_status(void);
extern void ipic_clear_mcp_status(u32 mask);

18
arch/powerpc/include/asm/irq.h
View File

@ -48,23 +48,19 @@ struct pt_regs;
* Per-cpu stacks for handling critical, debug and machine check
* level interrupts.
*/
extern struct thread_info *critirq_ctx[NR_CPUS];
extern struct thread_info *dbgirq_ctx[NR_CPUS];
extern struct thread_info *mcheckirq_ctx[NR_CPUS];
extern void exc_lvl_ctx_init(void);
#else
#define exc_lvl_ctx_init()
extern void *critirq_ctx[NR_CPUS];
extern void *dbgirq_ctx[NR_CPUS];
extern void *mcheckirq_ctx[NR_CPUS];
#endif
/*
* Per-cpu stacks for handling hard and soft interrupts.
*/
extern struct thread_info *hardirq_ctx[NR_CPUS];
extern struct thread_info *softirq_ctx[NR_CPUS];
extern void *hardirq_ctx[NR_CPUS];
extern void *softirq_ctx[NR_CPUS];
extern void irq_ctx_init(void);
extern void call_do_softirq(struct thread_info *tp);
extern void call_do_irq(struct pt_regs *regs, struct thread_info *tp);
void call_do_softirq(void *sp);
void call_do_irq(struct pt_regs *regs, void *sp);
extern void do_IRQ(struct pt_regs *regs);
extern void __init init_IRQ(void);
extern void __do_irq(struct pt_regs *regs);

3
arch/powerpc/include/asm/kvm_ppc.h
View File

@ -141,6 +141,7 @@ extern void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu);
extern int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu);
extern int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu);
extern void kvmppc_core_queue_machine_check(struct kvm_vcpu *vcpu, ulong flags);
extern void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong flags);
extern void kvmppc_core_queue_fpunavail(struct kvm_vcpu *vcpu);
extern void kvmppc_core_queue_vec_unavail(struct kvm_vcpu *vcpu);
@ -632,7 +633,7 @@ long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target,
unsigned int yield_count);
long kvmppc_h_random(struct kvm_vcpu *vcpu);
void kvmhv_commence_exit(int trap);
long kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu);
void kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu);
void kvmppc_subcore_enter_guest(void);
void kvmppc_subcore_exit_guest(void);
long kvmppc_realmode_hmi_handler(void);

7
arch/powerpc/include/asm/livepatch.h
View File

@ -21,6 +21,7 @@
#include <linux/module.h>
#include <linux/ftrace.h>
#include <linux/sched/task_stack.h>
#ifdef CONFIG_LIVEPATCH
static inline int klp_check_compiler_support(void)
@ -43,13 +44,13 @@ static inline unsigned long klp_get_ftrace_location(unsigned long faddr)
return ftrace_location_range(faddr, faddr + 16);
}
static inline void klp_init_thread_info(struct thread_info *ti)
static inline void klp_init_thread_info(struct task_struct *p)
{
/* + 1 to account for STACK_END_MAGIC */
ti->livepatch_sp = (unsigned long *)(ti + 1) + 1;
task_thread_info(p)->livepatch_sp = end_of_stack(p) + 1;
}
#else
static void klp_init_thread_info(struct thread_info *ti) { }
static inline void klp_init_thread_info(struct task_struct *p) { }
#endif /* CONFIG_LIVEPATCH */
#endif /* _ASM_POWERPC_LIVEPATCH_H */

4
arch/powerpc/include/asm/machdep.h
View File

@ -47,9 +47,7 @@ struct machdep_calls {
#endif
#endif /* CONFIG_PPC64 */
/* Platform set_dma_mask and dma_get_required_mask overrides */
int (*dma_set_mask)(struct device *dev, u64 dma_mask);
u64 (*dma_get_required_mask)(struct device *dev);
void (*dma_set_mask)(struct device *dev, u64 dma_mask);
int (*probe)(void);
void (*setup_arch)(void); /* Optional, may be NULL */

2
arch/powerpc/include/asm/mce.h
View File

@ -209,7 +209,7 @@ extern int get_mce_event(struct machine_check_event *mce, bool release);
extern void release_mce_event(void);
extern void machine_check_queue_event(void);
extern void machine_check_print_event_info(struct machine_check_event *evt,
bool user_mode);
bool user_mode, bool in_guest);
#ifdef CONFIG_PPC_BOOK3S_64
void flush_and_reload_slb(void);
#endif /* CONFIG_PPC_BOOK3S_64 */

13
arch/powerpc/include/asm/mmu.h
View File

@ -289,6 +289,17 @@ static inline u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address)
}
#endif /* CONFIG_PPC_MEM_KEYS */
#ifdef CONFIG_STRICT_KERNEL_RWX
static inline bool strict_kernel_rwx_enabled(void)
{
return rodata_enabled;
}
#else
static inline bool strict_kernel_rwx_enabled(void)
{
return false;
}
#endif
#endif /* !__ASSEMBLY__ */
/* The kernel use the constants below to index in the page sizes array.
@ -356,6 +367,8 @@ extern void early_init_mmu_secondary(void);
extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
phys_addr_t first_memblock_size);
static inline void mmu_early_init_devtree(void) { }
extern void *abatron_pteptrs[2];
#endif /* __ASSEMBLY__ */
#endif

2
arch/powerpc/include/asm/nmi.h
View File

@ -14,4 +14,6 @@ extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask,
#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace
#endif
extern void hv_nmi_check_nonrecoverable(struct pt_regs *regs);
#endif /* _ASM_NMI_H */

3
arch/powerpc/include/asm/nohash/32/mmu-8xx.h
View File

@ -231,9 +231,10 @@ static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
}
/* patch sites */
extern s32 patch__itlbmiss_linmem_top;
extern s32 patch__itlbmiss_linmem_top, patch__itlbmiss_linmem_top8;
extern s32 patch__dtlbmiss_linmem_top, patch__dtlbmiss_immr_jmp;
extern s32 patch__fixupdar_linmem_top;
extern s32 patch__dtlbmiss_romem_top, patch__dtlbmiss_romem_top8;
extern s32 patch__itlbmiss_exit_1, patch__itlbmiss_exit_2;
extern s32 patch__dtlbmiss_exit_1, patch__dtlbmiss_exit_2, patch__dtlbmiss_exit_3;

14
arch/powerpc/include/asm/page.h
View File

@ -20,20 +20,11 @@
/*
* On regular PPC32 page size is 4K (but we support 4K/16K/64K/256K pages
* on PPC44x). For PPC64 we support either 4K or 64K software
* on PPC44x and 4K/16K on 8xx). For PPC64 we support either 4K or 64K software
* page size. When using 64K pages however, whether we are really supporting
* 64K pages in HW or not is irrelevant to those definitions.
*/
#if defined(CONFIG_PPC_256K_PAGES)
#define PAGE_SHIFT 18
#elif defined(CONFIG_PPC_64K_PAGES)
#define PAGE_SHIFT 16
#elif defined(CONFIG_PPC_16K_PAGES)
#define PAGE_SHIFT 14
#else
#define PAGE_SHIFT 12
#endif
#define PAGE_SHIFT CONFIG_PPC_PAGE_SHIFT
#define PAGE_SIZE (ASM_CONST(1) << PAGE_SHIFT)
#ifndef __ASSEMBLY__
@ -326,7 +317,6 @@ struct page;
extern void clear_user_page(void *page, unsigned long vaddr, struct page *pg);
extern void copy_user_page(void *to, void *from, unsigned long vaddr,
struct page *p);
extern int page_is_ram(unsigned long pfn);
extern int devmem_is_allowed(unsigned long pfn);
#ifdef CONFIG_PPC_SMLPAR

7
arch/powerpc/include/asm/pci-bridge.h
View File

@ -20,6 +20,8 @@ struct device_node;
struct pci_controller_ops {
void (*dma_dev_setup)(struct pci_dev *pdev);
void (*dma_bus_setup)(struct pci_bus *bus);
bool (*iommu_bypass_supported)(struct pci_dev *pdev,
u64 mask);
int (*probe_mode)(struct pci_bus *bus);
@ -44,9 +46,6 @@ struct pci_controller_ops {
void (*teardown_msi_irqs)(struct pci_dev *pdev);
#endif
int (*dma_set_mask)(struct pci_dev *pdev, u64 dma_mask);
u64 (*dma_get_required_mask)(struct pci_dev *pdev);
void (*shutdown)(struct pci_controller *hose);
};
@ -275,6 +274,8 @@ extern int pcibios_map_io_space(struct pci_bus *bus);
extern struct pci_controller *pci_find_hose_for_OF_device(
struct device_node* node);
extern struct pci_controller *pci_find_controller_for_domain(int domain_nr);
/* Fill up host controller resources from the OF node */
extern void pci_process_bridge_OF_ranges(struct pci_controller *hose,
struct device_node *dev, int primary);

2
arch/powerpc/include/asm/pci.h
View File

@ -52,10 +52,8 @@ static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel)
#ifdef CONFIG_PCI
extern void set_pci_dma_ops(const struct dma_map_ops *dma_ops);
extern const struct dma_map_ops *get_pci_dma_ops(void);
#else /* CONFIG_PCI */
#define set_pci_dma_ops(d)
#define get_pci_dma_ops() NULL
#endif
#ifdef CONFIG_PPC64

1
arch/powerpc/include/asm/pgtable.h
View File

@ -66,7 +66,6 @@ extern unsigned long empty_zero_page[];
extern pgd_t swapper_pg_dir[];
int dma_pfn_limit_to_zone(u64 pfn_limit);
extern void paging_init(void);
/*

3
arch/powerpc/include/asm/powernv.h
View File

@ -23,6 +23,8 @@ extern int pnv_npu2_handle_fault(struct npu_context *context, uintptr_t *ea,
unsigned long *flags, unsigned long *status,
int count);
void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val);
void pnv_tm_init(void);
#else
static inline void powernv_set_nmmu_ptcr(unsigned long ptcr) { }
@ -40,7 +42,6 @@ static inline int pnv_npu2_handle_fault(struct npu_context *context,
}
static inline void pnv_tm_init(void) { }
static inline void pnv_power9_force_smt4(void) { }
#endif
#endif /* _ASM_POWERNV_H */

16
arch/powerpc/include/asm/ppc-opcode.h
View File

@ -326,6 +326,7 @@
#define PPC_INST_ADDI 0x38000000
#define PPC_INST_ADDIS 0x3c000000
#define PPC_INST_ADD 0x7c000214
#define PPC_INST_ADDC 0x7c000014
#define PPC_INST_SUB 0x7c000050
#define PPC_INST_BLR 0x4e800020
#define PPC_INST_BLRL 0x4e800021
@ -334,6 +335,9 @@
#define PPC_INST_MULLW 0x7c0001d6
#define PPC_INST_MULHWU 0x7c000016
#define PPC_INST_MULLI 0x1c000000
#define PPC_INST_MADDHD 0x10000030
#define PPC_INST_MADDHDU 0x10000031
#define PPC_INST_MADDLD 0x10000033
#define PPC_INST_DIVWU 0x7c000396
#define PPC_INST_DIVD 0x7c0003d2
#define PPC_INST_RLWINM 0x54000000
@ -377,6 +381,7 @@
/* macros to insert fields into opcodes */
#define ___PPC_RA(a) (((a) & 0x1f) << 16)
#define ___PPC_RB(b) (((b) & 0x1f) << 11)
#define ___PPC_RC(c) (((c) & 0x1f) << 6)
#define ___PPC_RS(s) (((s) & 0x1f) << 21)
#define ___PPC_RT(t) ___PPC_RS(t)
#define ___PPC_R(r) (((r) & 0x1) << 16)
@ -396,7 +401,7 @@
#define __PPC_WS(w) (((w) & 0x1f) << 11)
#define __PPC_SH(s) __PPC_WS(s)
#define __PPC_SH64(s) (__PPC_SH(s) | (((s) & 0x20) >> 4))
#define __PPC_MB(s) (((s) & 0x1f) << 6)
#define __PPC_MB(s) ___PPC_RC(s)
#define __PPC_ME(s) (((s) & 0x1f) << 1)
#define __PPC_MB64(s) (__PPC_MB(s) | ((s) & 0x20))
#define __PPC_ME64(s) __PPC_MB64(s)
@ -438,6 +443,15 @@
#define PPC_STQCX(t, a, b) stringify_in_c(.long PPC_INST_STQCX | \
___PPC_RT(t) | ___PPC_RA(a) | \
___PPC_RB(b))
#define PPC_MADDHD(t, a, b, c) stringify_in_c(.long PPC_INST_MADDHD | \
___PPC_RT(t) | ___PPC_RA(a) | \
___PPC_RB(b) | ___PPC_RC(c))
#define PPC_MADDHDU(t, a, b, c) stringify_in_c(.long PPC_INST_MADDHDU | \
___PPC_RT(t) | ___PPC_RA(a) | \
___PPC_RB(b) | ___PPC_RC(c))
#define PPC_MADDLD(t, a, b, c) stringify_in_c(.long PPC_INST_MADDLD | \
___PPC_RT(t) | ___PPC_RA(a) | \
___PPC_RB(b) | ___PPC_RC(c))
#define PPC_MSGSND(b) stringify_in_c(.long PPC_INST_MSGSND | \
___PPC_RB(b))
#define PPC_MSGSYNC stringify_in_c(.long PPC_INST_MSGSYNC)

4
arch/powerpc/include/asm/ppc-pci.h
View File

@ -53,13 +53,13 @@ void eeh_addr_cache_rmv_dev(struct pci_dev *dev);
struct eeh_dev *eeh_addr_cache_get_dev(unsigned long addr);
void eeh_slot_error_detail(struct eeh_pe *pe, int severity);
int eeh_pci_enable(struct eeh_pe *pe, int function);
int eeh_pe_reset_full(struct eeh_pe *pe);
int eeh_pe_reset_full(struct eeh_pe *pe, bool include_passed);
void eeh_save_bars(struct eeh_dev *edev);
int rtas_write_config(struct pci_dn *, int where, int size, u32 val);
int rtas_read_config(struct pci_dn *, int where, int size, u32 *val);
void eeh_pe_state_mark(struct eeh_pe *pe, int state);
void eeh_pe_mark_isolated(struct eeh_pe *pe);
void eeh_pe_state_clear(struct eeh_pe *pe, int state);
void eeh_pe_state_clear(struct eeh_pe *pe, int state, bool include_passed);
void eeh_pe_state_mark_with_cfg(struct eeh_pe *pe, int state);
void eeh_pe_dev_mode_mark(struct eeh_pe *pe, int mode);

110
arch/powerpc/include/asm/processor.h
View File

@ -40,7 +40,7 @@
#ifndef __ASSEMBLY__
#include <linux/types.h>
#include <asm/thread_info.h>
#include <linux/thread_info.h>
#include <asm/ptrace.h>
#include <asm/hw_breakpoint.h>
@ -77,106 +77,16 @@ extern int _chrp_type;
#ifdef __KERNEL__
#ifdef CONFIG_PPC64
#include <asm/task_size_64.h>
#else
#include <asm/task_size_32.h>
#endif
struct task_struct;
void start_thread(struct pt_regs *regs, unsigned long fdptr, unsigned long sp);
void release_thread(struct task_struct *);
#ifdef CONFIG_PPC32
#if CONFIG_TASK_SIZE > CONFIG_KERNEL_START
#error User TASK_SIZE overlaps with KERNEL_START address
#endif
#define TASK_SIZE (CONFIG_TASK_SIZE)
/* This decides where the kernel will search for a free chunk of vm
* space during mmap's.
*/
#define TASK_UNMAPPED_BASE (TASK_SIZE / 8 * 3)
#endif
#ifdef CONFIG_PPC64
/*
* 64-bit user address space can have multiple limits
* For now supported values are:
*/
#define TASK_SIZE_64TB (0x0000400000000000UL)
#define TASK_SIZE_128TB (0x0000800000000000UL)
#define TASK_SIZE_512TB (0x0002000000000000UL)
#define TASK_SIZE_1PB (0x0004000000000000UL)
#define TASK_SIZE_2PB (0x0008000000000000UL)
/*
* With 52 bits in the address we can support
* upto 4PB of range.
*/
#define TASK_SIZE_4PB (0x0010000000000000UL)
/*
* For now 512TB is only supported with book3s and 64K linux page size.
*/
#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_PPC_64K_PAGES)
/*
* Max value currently used:
*/
#define TASK_SIZE_USER64 TASK_SIZE_4PB
#define DEFAULT_MAP_WINDOW_USER64 TASK_SIZE_128TB
#define TASK_CONTEXT_SIZE TASK_SIZE_512TB
#else
#define TASK_SIZE_USER64 TASK_SIZE_64TB
#define DEFAULT_MAP_WINDOW_USER64 TASK_SIZE_64TB
/*
* We don't need to allocate extended context ids for 4K page size, because
* we limit the max effective address on this config to 64TB.
*/
#define TASK_CONTEXT_SIZE TASK_SIZE_64TB
#endif
/*
* 32-bit user address space is 4GB - 1 page
* (this 1 page is needed so referencing of 0xFFFFFFFF generates EFAULT
*/
#define TASK_SIZE_USER32 (0x0000000100000000UL - (1*PAGE_SIZE))
#define TASK_SIZE_OF(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT) ? \
TASK_SIZE_USER32 : TASK_SIZE_USER64)
#define TASK_SIZE TASK_SIZE_OF(current)
/* This decides where the kernel will search for a free chunk of vm
* space during mmap's.
*/
#define TASK_UNMAPPED_BASE_USER32 (PAGE_ALIGN(TASK_SIZE_USER32 / 4))
#define TASK_UNMAPPED_BASE_USER64 (PAGE_ALIGN(DEFAULT_MAP_WINDOW_USER64 / 4))
#define TASK_UNMAPPED_BASE ((is_32bit_task()) ? \
TASK_UNMAPPED_BASE_USER32 : TASK_UNMAPPED_BASE_USER64 )
#endif
/*
* Initial task size value for user applications. For book3s 64 we start
* with 128TB and conditionally enable upto 512TB
*/
#ifdef CONFIG_PPC_BOOK3S_64
#define DEFAULT_MAP_WINDOW ((is_32bit_task()) ? \
TASK_SIZE_USER32 : DEFAULT_MAP_WINDOW_USER64)
#else
#define DEFAULT_MAP_WINDOW TASK_SIZE
#endif
#ifdef __powerpc64__
#define STACK_TOP_USER64 DEFAULT_MAP_WINDOW_USER64
#define STACK_TOP_USER32 TASK_SIZE_USER32
#define STACK_TOP (is_32bit_task() ? \
STACK_TOP_USER32 : STACK_TOP_USER64)
#define STACK_TOP_MAX TASK_SIZE_USER64
#else /* __powerpc64__ */
#define STACK_TOP TASK_SIZE
#define STACK_TOP_MAX STACK_TOP
#endif /* __powerpc64__ */
typedef struct {
unsigned long seg;
} mm_segment_t;
@ -250,6 +160,9 @@ struct thread_struct {
#ifdef CONFIG_PPC32
void *pgdir; /* root of page-table tree */
unsigned long ksp_limit; /* if ksp <= ksp_limit stack overflow */
#ifdef CONFIG_PPC_RTAS
unsigned long rtas_sp; /* stack pointer for when in RTAS */
#endif
#endif
/* Debug Registers */
struct debug_reg debug;
@ -357,8 +270,7 @@ struct thread_struct {
#define ARCH_MIN_TASKALIGN 16
#define INIT_SP (sizeof(init_stack) + (unsigned long) &init_stack)
#define INIT_SP_LIMIT \
(_ALIGN_UP(sizeof(init_thread_info), 16) + (unsigned long) &init_stack)
#define INIT_SP_LIMIT ((unsigned long)&init_stack)
#ifdef CONFIG_SPE
#define SPEFSCR_INIT \

2
arch/powerpc/include/asm/ptrace.h
View File

@ -157,7 +157,7 @@ extern int ptrace_put_reg(struct task_struct *task, int regno,
unsigned long data);
#define current_pt_regs() \
((struct pt_regs *)((unsigned long)current_thread_info() + THREAD_SIZE) - 1)
((struct pt_regs *)((unsigned long)task_stack_page(current) + THREAD_SIZE) - 1)
/*
* We use the least-significant bit of the trap field to indicate
* whether we have saved the full set of registers, or only a

9
arch/powerpc/include/asm/reg.h
View File

@ -1062,7 +1062,7 @@
* - SPRG9 debug exception scratch
*
* All 32-bit:
* - SPRG3 current thread_info pointer
* - SPRG3 current thread_struct physical addr pointer
* (virtual on BookE, physical on others)
*
* 32-bit classic:
@ -1167,7 +1167,7 @@
#ifdef CONFIG_PPC_BOOK3S_32
#define SPRN_SPRG_SCRATCH0 SPRN_SPRG0
#define SPRN_SPRG_SCRATCH1 SPRN_SPRG1
#define SPRN_SPRG_RTAS SPRN_SPRG2
#define SPRN_SPRG_PGDIR SPRN_SPRG2
#define SPRN_SPRG_603_LRU SPRN_SPRG4
#endif
@ -1425,6 +1425,11 @@ static inline void msr_check_and_clear(unsigned long bits)
#define mfsrin(v) ({unsigned int rval; \
asm volatile("mfsrin %0,%1" : "=r" (rval) : "r" (v)); \
rval;})
static inline void mtsrin(u32 val, u32 idx)
{
asm volatile("mtsrin %0, %1" : : "r" (val), "r" (idx));
}
#endif
#define proc_trap() asm volatile("trap")

7
arch/powerpc/include/asm/sections.h
View File

@ -17,6 +17,13 @@ extern char __end_interrupts[];
extern char __prom_init_toc_start[];
extern char __prom_init_toc_end[];
#ifdef CONFIG_PPC_POWERNV
extern char start_real_trampolines[];
extern char end_real_trampolines[];
extern char start_virt_trampolines[];
extern char end_virt_trampolines[];
#endif
static inline int in_kernel_text(unsigned long addr)
{
if (addr >= (unsigned long)_stext && addr < (unsigned long)__init_end)

17
arch/powerpc/include/asm/smp.h
View File

@ -83,7 +83,22 @@ int is_cpu_dead(unsigned int cpu);
/* 32-bit */
extern int smp_hw_index[];
#define raw_smp_processor_id() (current_thread_info()->cpu)
/*
* This is particularly ugly: it appears we can't actually get the definition
* of task_struct here, but we need access to the CPU this task is running on.
* Instead of using task_struct we're using _TASK_CPU which is extracted from
* asm-offsets.h by kbuild to get the current processor ID.
*
* This also needs to be safeguarded when building asm-offsets.s because at
* that time _TASK_CPU is not defined yet. It could have been guarded by
* _TASK_CPU itself, but we want the build to fail if _TASK_CPU is missing
* when building something else than asm-offsets.s
*/
#ifdef GENERATING_ASM_OFFSETS
#define raw_smp_processor_id() (0)
#else
#define raw_smp_processor_id() (*(unsigned int *)((void *)current + _TASK_CPU))
#endif
#define hard_smp_processor_id() (smp_hw_index[smp_processor_id()])
static inline int get_hard_smp_processor_id(int cpu)

5
arch/powerpc/include/asm/swiotlb.h
View File

@ -13,12 +13,7 @@
#include <linux/swiotlb.h>
extern const struct dma_map_ops powerpc_swiotlb_dma_ops;
extern unsigned int ppc_swiotlb_enable;
int __init swiotlb_setup_bus_notifier(void);
extern void pci_dma_dev_setup_swiotlb(struct pci_dev *pdev);
#ifdef CONFIG_SWIOTLB
void swiotlb_detect_4g(void);

21
arch/powerpc/include/asm/task_size_32.h Normal file
View File

@ -0,0 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_TASK_SIZE_32_H
#define _ASM_POWERPC_TASK_SIZE_32_H
#if CONFIG_TASK_SIZE > CONFIG_KERNEL_START
#error User TASK_SIZE overlaps with KERNEL_START address
#endif
#define TASK_SIZE (CONFIG_TASK_SIZE)
/*
* This decides where the kernel will search for a free chunk of vm space during
* mmap's.
*/
#define TASK_UNMAPPED_BASE (TASK_SIZE / 8 * 3)
#define DEFAULT_MAP_WINDOW TASK_SIZE
#define STACK_TOP TASK_SIZE
#define STACK_TOP_MAX STACK_TOP
#endif /* _ASM_POWERPC_TASK_SIZE_32_H */

79
arch/powerpc/include/asm/task_size_64.h Normal file
View File

@ -0,0 +1,79 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_TASK_SIZE_64_H
#define _ASM_POWERPC_TASK_SIZE_64_H
/*
* 64-bit user address space can have multiple limits
* For now supported values are:
*/
#define TASK_SIZE_64TB (0x0000400000000000UL)
#define TASK_SIZE_128TB (0x0000800000000000UL)
#define TASK_SIZE_512TB (0x0002000000000000UL)
#define TASK_SIZE_1PB (0x0004000000000000UL)
#define TASK_SIZE_2PB (0x0008000000000000UL)
/*
* With 52 bits in the address we can support up to 4PB of range.
*/
#define TASK_SIZE_4PB (0x0010000000000000UL)
/*
* For now 512TB is only supported with book3s and 64K linux page size.
*/
#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_PPC_64K_PAGES)
/*
* Max value currently used:
*/
#define TASK_SIZE_USER64 TASK_SIZE_4PB
#define DEFAULT_MAP_WINDOW_USER64 TASK_SIZE_128TB
#define TASK_CONTEXT_SIZE TASK_SIZE_512TB
#else
#define TASK_SIZE_USER64 TASK_SIZE_64TB
#define DEFAULT_MAP_WINDOW_USER64 TASK_SIZE_64TB
/*
* We don't need to allocate extended context ids for 4K page size, because we
* limit the max effective address on this config to 64TB.
*/
#define TASK_CONTEXT_SIZE TASK_SIZE_64TB
#endif
/*
* 32-bit user address space is 4GB - 1 page
* (this 1 page is needed so referencing of 0xFFFFFFFF generates EFAULT
*/
#define TASK_SIZE_USER32 (0x0000000100000000UL - (1 * PAGE_SIZE))
#define TASK_SIZE_OF(tsk) \
(test_tsk_thread_flag(tsk, TIF_32BIT) ? TASK_SIZE_USER32 : \
TASK_SIZE_USER64)
#define TASK_SIZE TASK_SIZE_OF(current)
#define TASK_UNMAPPED_BASE_USER32 (PAGE_ALIGN(TASK_SIZE_USER32 / 4))
#define TASK_UNMAPPED_BASE_USER64 (PAGE_ALIGN(DEFAULT_MAP_WINDOW_USER64 / 4))
/*
* This decides where the kernel will search for a free chunk of vm space during
* mmap's.
*/
#define TASK_UNMAPPED_BASE \
((is_32bit_task()) ? TASK_UNMAPPED_BASE_USER32 : TASK_UNMAPPED_BASE_USER64)
/*
* Initial task size value for user applications. For book3s 64 we start
* with 128TB and conditionally enable upto 512TB
*/
#ifdef CONFIG_PPC_BOOK3S_64
#define DEFAULT_MAP_WINDOW \
((is_32bit_task()) ? TASK_SIZE_USER32 : DEFAULT_MAP_WINDOW_USER64)
#else
#define DEFAULT_MAP_WINDOW TASK_SIZE
#endif
#define STACK_TOP_USER64 DEFAULT_MAP_WINDOW_USER64
#define STACK_TOP_USER32 TASK_SIZE_USER32
#define STACK_TOP_MAX TASK_SIZE_USER64
#define STACK_TOP (is_32bit_task() ? STACK_TOP_USER32 : STACK_TOP_USER64)
#endif /* _ASM_POWERPC_TASK_SIZE_64_H */

19
arch/powerpc/include/asm/thread_info.h
View File

@ -17,12 +17,6 @@
#define THREAD_SIZE (1 << THREAD_SHIFT)
#ifdef CONFIG_PPC64
#define CURRENT_THREAD_INFO(dest, sp) stringify_in_c(clrrdi dest, sp, THREAD_SHIFT)
#else
#define CURRENT_THREAD_INFO(dest, sp) stringify_in_c(rlwinm dest, sp, 0, 0, 31-THREAD_SHIFT)
#endif
#ifndef __ASSEMBLY__
#include <linux/cache.h>
#include <asm/processor.h>
@ -34,8 +28,6 @@
* low level task data.
*/
struct thread_info {
struct task_struct *task; /* main task structure */
int cpu; /* cpu we're on */
int preempt_count; /* 0 => preemptable,
<0 => BUG */
unsigned long local_flags; /* private flags for thread */
@ -58,8 +50,6 @@ struct thread_info {
*/
#define INIT_THREAD_INFO(tsk) \
{ \
.task = &tsk, \
.cpu = 0, \
.preempt_count = INIT_PREEMPT_COUNT, \
.flags = 0, \
}
@ -67,15 +57,6 @@ struct thread_info {
#define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT)
/* how to get the thread information struct from C */
static inline struct thread_info *current_thread_info(void)
{
unsigned long val;
asm (CURRENT_THREAD_INFO(%0,1) : "=r" (val));
return (struct thread_info *)val;
}
extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
#ifdef CONFIG_PPC_BOOK3S_64

2
arch/powerpc/include/asm/topology.h
View File

@ -132,6 +132,8 @@ static inline void shared_proc_topology_init(void) {}
#define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu))
#define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu))
#define topology_core_id(cpu) (cpu_to_core_id(cpu))
int dlpar_cpu_readd(int cpu);
#endif
#endif

15
arch/powerpc/kernel/Makefile
View File

@ -36,7 +36,7 @@ obj-y := cputable.o ptrace.o syscalls.o \
process.o systbl.o idle.o \
signal.o sysfs.o cacheinfo.o time.o \
prom.o traps.o setup-common.o \
udbg.o misc.o io.o dma.o misc_$(BITS).o \
udbg.o misc.o io.o misc_$(BITS).o \
of_platform.o prom_parse.o
obj-$(CONFIG_PPC64) += setup_64.o sys_ppc32.o \
signal_64.o ptrace32.o \
@ -105,6 +105,7 @@ obj-$(CONFIG_UPROBES) += uprobes.o
obj-$(CONFIG_PPC_UDBG_16550) += legacy_serial.o udbg_16550.o
obj-$(CONFIG_STACKTRACE) += stacktrace.o
obj-$(CONFIG_SWIOTLB) += dma-swiotlb.o
obj-$(CONFIG_ARCH_HAS_DMA_SET_MASK) += dma-mask.o
pci64-$(CONFIG_PPC64) += pci_dn.o pci-hotplug.o isa-bridge.o
obj-$(CONFIG_PCI) += pci_$(BITS).o $(pci64-y) \
@ -142,19 +143,29 @@ endif
obj-$(CONFIG_EPAPR_PARAVIRT) += epapr_paravirt.o epapr_hcalls.o
obj-$(CONFIG_KVM_GUEST) += kvm.o kvm_emul.o
# Disable GCOV & sanitizers in odd or sensitive code
# Disable GCOV, KCOV & sanitizers in odd or sensitive code
GCOV_PROFILE_prom_init.o := n
KCOV_INSTRUMENT_prom_init.o := n
UBSAN_SANITIZE_prom_init.o := n
GCOV_PROFILE_machine_kexec_64.o := n
KCOV_INSTRUMENT_machine_kexec_64.o := n
UBSAN_SANITIZE_machine_kexec_64.o := n
GCOV_PROFILE_machine_kexec_32.o := n
KCOV_INSTRUMENT_machine_kexec_32.o := n
UBSAN_SANITIZE_machine_kexec_32.o := n
GCOV_PROFILE_kprobes.o := n
KCOV_INSTRUMENT_kprobes.o := n
UBSAN_SANITIZE_kprobes.o := n
GCOV_PROFILE_kprobes-ftrace.o := n
KCOV_INSTRUMENT_kprobes-ftrace.o := n
UBSAN_SANITIZE_kprobes-ftrace.o := n
UBSAN_SANITIZE_vdso.o := n
# Necessary for booting with kcov enabled on book3e machines
KCOV_INSTRUMENT_cputable.o := n
KCOV_INSTRUMENT_setup_64.o := n
KCOV_INSTRUMENT_paca.o := n
extra-$(CONFIG_PPC_FPU) += fpu.o
extra-$(CONFIG_ALTIVEC) += vector.o
extra-$(CONFIG_PPC64) += entry_64.o

15
arch/powerpc/kernel/asm-offsets.c
View File

@ -13,6 +13,8 @@
* 2 of the License, or (at your option) any later version.
*/
#define GENERATING_ASM_OFFSETS /* asm/smp.h */
#include <linux/compat.h>
#include <linux/signal.h>
#include <linux/sched.h>
@ -90,10 +92,15 @@ int main(void)
DEFINE(SIGSEGV, SIGSEGV);
DEFINE(NMI_MASK, NMI_MASK);
#else
OFFSET(THREAD_INFO, task_struct, stack);
DEFINE(THREAD_INFO_GAP, _ALIGN_UP(sizeof(struct thread_info), 16));
OFFSET(KSP_LIMIT, thread_struct, ksp_limit);
#ifdef CONFIG_PPC_RTAS
OFFSET(RTAS_SP, thread_struct, rtas_sp);
#endif
#endif /* CONFIG_PPC64 */
OFFSET(TASK_STACK, task_struct, stack);
#ifdef CONFIG_SMP
OFFSET(TASK_CPU, task_struct, cpu);
#endif
#ifdef CONFIG_LIVEPATCH
OFFSET(TI_livepatch_sp, thread_info, livepatch_sp);
@ -161,8 +168,6 @@ int main(void)
OFFSET(TI_FLAGS, thread_info, flags);
OFFSET(TI_LOCAL_FLAGS, thread_info, local_flags);
OFFSET(TI_PREEMPT, thread_info, preempt_count);
OFFSET(TI_TASK, thread_info, task);
OFFSET(TI_CPU, thread_info, cpu);
#ifdef CONFIG_PPC64
OFFSET(DCACHEL1BLOCKSIZE, ppc64_caches, l1d.block_size);
@ -177,6 +182,8 @@ int main(void)
OFFSET(PACAPROCSTART, paca_struct, cpu_start);
OFFSET(PACAKSAVE, paca_struct, kstack);
OFFSET(PACACURRENT, paca_struct, __current);
DEFINE(PACA_THREAD_INFO, offsetof(struct paca_struct, __current) +
offsetof(struct task_struct, thread_info));
OFFSET(PACASAVEDMSR, paca_struct, saved_msr);
OFFSET(PACAR1, paca_struct, saved_r1);
OFFSET(PACATOC, paca_struct, kernel_toc);

4
arch/powerpc/kernel/cpu_setup_6xx.S
View File

@ -24,6 +24,10 @@ BEGIN_MMU_FTR_SECTION
li r10,0
mtspr SPRN_SPRG_603_LRU,r10 /* init SW LRU tracking */
END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
lis r10, (swapper_pg_dir - PAGE_OFFSET)@h
ori r10, r10, (swapper_pg_dir - PAGE_OFFSET)@l
mtspr SPRN_SPRG_PGDIR, r10
BEGIN_FTR_SECTION
bl __init_fpu_registers
END_FTR_SECTION_IFCLR(CPU_FTR_FPU_UNAVAILABLE)

75
arch/powerpc/kernel/dma-iommu.c
View File

@ -6,12 +6,31 @@
* busses using the iommu infrastructure
*/
#include <linux/dma-direct.h>
#include <linux/pci.h>
#include <asm/iommu.h>
/*
* Generic iommu implementation
*/
/*
* The coherent mask may be smaller than the real mask, check if we can
* really use a direct window.
*/
static inline bool dma_iommu_alloc_bypass(struct device *dev)
{
return dev->archdata.iommu_bypass && !iommu_fixed_is_weak &&
dma_direct_supported(dev, dev->coherent_dma_mask);
}
static inline bool dma_iommu_map_bypass(struct device *dev,
unsigned long attrs)
{
return dev->archdata.iommu_bypass &&
(!iommu_fixed_is_weak || (attrs & DMA_ATTR_WEAK_ORDERING));
}
/* Allocates a contiguous real buffer and creates mappings over it.
* Returns the virtual address of the buffer and sets dma_handle
* to the dma address (mapping) of the first page.
@ -20,6 +39,8 @@ static void *dma_iommu_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
unsigned long attrs)
{
if (dma_iommu_alloc_bypass(dev))
return dma_direct_alloc(dev, size, dma_handle, flag, attrs);
return iommu_alloc_coherent(dev, get_iommu_table_base(dev), size,
dma_handle, dev->coherent_dma_mask, flag,
dev_to_node(dev));
@ -29,7 +50,11 @@ static void dma_iommu_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
iommu_free_coherent(get_iommu_table_base(dev), size, vaddr, dma_handle);
if (dma_iommu_alloc_bypass(dev))
dma_direct_free(dev, size, vaddr, dma_handle, attrs);
else
iommu_free_coherent(get_iommu_table_base(dev), size, vaddr,
dma_handle);
}
/* Creates TCEs for a user provided buffer. The user buffer must be
@ -42,6 +67,9 @@ static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page,
enum dma_data_direction direction,
unsigned long attrs)
{
if (dma_iommu_map_bypass(dev, attrs))
return dma_direct_map_page(dev, page, offset, size, direction,
attrs);
return iommu_map_page(dev, get_iommu_table_base(dev), page, offset,
size, device_to_mask(dev), direction, attrs);
}
@ -51,8 +79,9 @@ static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle,
size_t size, enum dma_data_direction direction,
unsigned long attrs)
{
iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction,
attrs);
if (!dma_iommu_map_bypass(dev, attrs))
iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size,
direction, attrs);
}
@ -60,6 +89,8 @@ static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction direction,
unsigned long attrs)
{
if (dma_iommu_map_bypass(dev, attrs))
return dma_direct_map_sg(dev, sglist, nelems, direction, attrs);
return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems,
device_to_mask(dev), direction, attrs);
}
@ -68,10 +99,20 @@ static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction direction,
unsigned long attrs)
{
ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
if (!dma_iommu_map_bypass(dev, attrs))
ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
direction, attrs);
}
static bool dma_iommu_bypass_supported(struct device *dev, u64 mask)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct pci_controller *phb = pci_bus_to_host(pdev->bus);
return phb->controller_ops.iommu_bypass_supported &&
phb->controller_ops.iommu_bypass_supported(pdev, mask);
}
/* We support DMA to/from any memory page via the iommu */
int dma_iommu_dma_supported(struct device *dev, u64 mask)
{
@ -83,32 +124,48 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask)
return 0;
}
if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) {
dev->archdata.iommu_bypass = true;
dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");
return 1;
}
if (tbl->it_offset > (mask >> tbl->it_page_shift)) {
dev_info(dev, "Warning: IOMMU offset too big for device mask\n");
dev_info(dev, "mask: 0x%08llx, table offset: 0x%08lx\n",
mask, tbl->it_offset << tbl->it_page_shift);
return 0;
} else
return 1;
}
dev_dbg(dev, "iommu: not 64-bit, using default ops\n");
dev->archdata.iommu_bypass = false;
return 1;
}
static u64 dma_iommu_get_required_mask(struct device *dev)
u64 dma_iommu_get_required_mask(struct device *dev)
{
struct iommu_table *tbl = get_iommu_table_base(dev);
u64 mask;
if (!tbl)
return 0;
if (dev_is_pci(dev)) {
u64 bypass_mask = dma_direct_get_required_mask(dev);
if (dma_iommu_bypass_supported(dev, bypass_mask))
return bypass_mask;
}
mask = 1ULL < (fls_long(tbl->it_offset + tbl->it_size) - 1);
mask += mask - 1;
return mask;
}
struct dma_map_ops dma_iommu_ops = {
const struct dma_map_ops dma_iommu_ops = {
.alloc = dma_iommu_alloc_coherent,
.free = dma_iommu_free_coherent,
.mmap = dma_nommu_mmap_coherent,
.map_sg = dma_iommu_map_sg,
.unmap_sg = dma_iommu_unmap_sg,
.dma_supported = dma_iommu_dma_supported,

12
arch/powerpc/kernel/dma-mask.c Normal file
View File

@ -0,0 +1,12 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/dma-mapping.h>
#include <linux/export.h>
#include <asm/machdep.h>
void arch_dma_set_mask(struct device *dev, u64 dma_mask)
{
if (ppc_md.dma_set_mask)
ppc_md.dma_set_mask(dev, dma_mask);
}
EXPORT_SYMBOL(arch_dma_set_mask);

89
arch/powerpc/kernel/dma-swiotlb.c
View File

@ -10,101 +10,12 @@
* option) any later version.
*
*/
#include <linux/dma-direct.h>
#include <linux/memblock.h>
#include <linux/pfn.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/pci.h>
#include <asm/machdep.h>
#include <asm/swiotlb.h>
#include <asm/dma.h>
unsigned int ppc_swiotlb_enable;
static u64 swiotlb_powerpc_get_required(struct device *dev)
{
u64 end, mask, max_direct_dma_addr = dev->archdata.max_direct_dma_addr;
end = memblock_end_of_DRAM();
if (max_direct_dma_addr && end > max_direct_dma_addr)
end = max_direct_dma_addr;
end += get_dma_offset(dev);
mask = 1ULL << (fls64(end) - 1);
mask += mask - 1;
return mask;
}
/*
* At the moment, all platforms that use this code only require
* swiotlb to be used if we're operating on HIGHMEM. Since
* we don't ever call anything other than map_sg, unmap_sg,
* map_page, and unmap_page on highmem, use normal dma_ops
* for everything else.
*/
const struct dma_map_ops powerpc_swiotlb_dma_ops = {
.alloc = __dma_nommu_alloc_coherent,
.free = __dma_nommu_free_coherent,
.mmap = dma_nommu_mmap_coherent,
.map_sg = dma_direct_map_sg,
.unmap_sg = dma_direct_unmap_sg,
.dma_supported = swiotlb_dma_supported,
.map_page = dma_direct_map_page,
.unmap_page = dma_direct_unmap_page,
.sync_single_for_cpu = dma_direct_sync_single_for_cpu,
.sync_single_for_device = dma_direct_sync_single_for_device,
.sync_sg_for_cpu = dma_direct_sync_sg_for_cpu,
.sync_sg_for_device = dma_direct_sync_sg_for_device,
.get_required_mask = swiotlb_powerpc_get_required,
};
void pci_dma_dev_setup_swiotlb(struct pci_dev *pdev)
{
struct pci_controller *hose;
struct dev_archdata *sd;
hose = pci_bus_to_host(pdev->bus);
sd = &pdev->dev.archdata;
sd->max_direct_dma_addr =
hose->dma_window_base_cur + hose->dma_window_size;
}
static int ppc_swiotlb_bus_notify(struct notifier_block *nb,
unsigned long action, void *data)
{
struct device *dev = data;
struct dev_archdata *sd;
/* We are only intereted in device addition */
if (action != BUS_NOTIFY_ADD_DEVICE)
return 0;
sd = &dev->archdata;
sd->max_direct_dma_addr = 0;
/* May need to bounce if the device can't address all of DRAM */
if ((dma_get_mask(dev) + 1) < memblock_end_of_DRAM())
set_dma_ops(dev, &powerpc_swiotlb_dma_ops);
return NOTIFY_DONE;
}
static struct notifier_block ppc_swiotlb_plat_bus_notifier = {
.notifier_call = ppc_swiotlb_bus_notify,
.priority = 0,
};
int __init swiotlb_setup_bus_notifier(void)
{
bus_register_notifier(&platform_bus_type,
&ppc_swiotlb_plat_bus_notifier);
return 0;
}
void __init swiotlb_detect_4g(void)
{
if ((memblock_end_of_DRAM() - 1) > 0xffffffff)

362
arch/powerpc/kernel/dma.c
View File

@ -1,362 +0,0 @@
/*
* Copyright (C) 2006 Benjamin Herrenschmidt, IBM Corporation
*
* Provide default implementations of the DMA mapping callbacks for
* directly mapped busses.
*/
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/dma-debug.h>
#include <linux/gfp.h>
#include <linux/memblock.h>
#include <linux/export.h>
#include <linux/pci.h>
#include <asm/vio.h>
#include <asm/bug.h>
#include <asm/machdep.h>
#include <asm/swiotlb.h>
#include <asm/iommu.h>
/*
* Generic direct DMA implementation
*
* This implementation supports a per-device offset that can be applied if
* the address at which memory is visible to devices is not 0. Platform code
* can set archdata.dma_data to an unsigned long holding the offset. By
* default the offset is PCI_DRAM_OFFSET.
*/
static u64 __maybe_unused get_pfn_limit(struct device *dev)
{
u64 pfn = (dev->coherent_dma_mask >> PAGE_SHIFT) + 1;
struct dev_archdata __maybe_unused *sd = &dev->archdata;
#ifdef CONFIG_SWIOTLB
if (sd->max_direct_dma_addr && dev->dma_ops == &powerpc_swiotlb_dma_ops)
pfn = min_t(u64, pfn, sd->max_direct_dma_addr >> PAGE_SHIFT);
#endif
return pfn;
}
static int dma_nommu_dma_supported(struct device *dev, u64 mask)
{
#ifdef CONFIG_PPC64
u64 limit = get_dma_offset(dev) + (memblock_end_of_DRAM() - 1);
/* Limit fits in the mask, we are good */
if (mask >= limit)
return 1;
#ifdef CONFIG_FSL_SOC
/*
* Freescale gets another chance via ZONE_DMA, however
* that will have to be refined if/when they support iommus
*/
return 1;
#endif
/* Sorry ... */
return 0;
#else
return 1;
#endif
}
#ifndef CONFIG_NOT_COHERENT_CACHE
void *__dma_nommu_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
unsigned long attrs)
{
void *ret;
struct page *page;
int node = dev_to_node(dev);
#ifdef CONFIG_FSL_SOC
u64 pfn = get_pfn_limit(dev);
int zone;
/*
* This code should be OK on other platforms, but we have drivers that
* don't set coherent_dma_mask. As a workaround we just ifdef it. This
* whole routine needs some serious cleanup.
*/
zone = dma_pfn_limit_to_zone(pfn);
if (zone < 0) {
dev_err(dev, "%s: No suitable zone for pfn %#llx\n",
__func__, pfn);
return NULL;
}
switch (zone) {
#ifdef CONFIG_ZONE_DMA
case ZONE_DMA:
flag |= GFP_DMA;
break;
#endif
};
#endif /* CONFIG_FSL_SOC */
page = alloc_pages_node(node, flag, get_order(size));
if (page == NULL)
return NULL;
ret = page_address(page);
memset(ret, 0, size);
*dma_handle = __pa(ret) + get_dma_offset(dev);
return ret;
}
void __dma_nommu_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
free_pages((unsigned long)vaddr, get_order(size));
}
#endif /* !CONFIG_NOT_COHERENT_CACHE */
static void *dma_nommu_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
unsigned long attrs)
{
struct iommu_table *iommu;
/* The coherent mask may be smaller than the real mask, check if
* we can really use the direct ops
*/
if (dma_nommu_dma_supported(dev, dev->coherent_dma_mask))
return __dma_nommu_alloc_coherent(dev, size, dma_handle,
flag, attrs);
/* Ok we can't ... do we have an iommu ? If not, fail */
iommu = get_iommu_table_base(dev);
if (!iommu)
return NULL;
/* Try to use the iommu */
return iommu_alloc_coherent(dev, iommu, size, dma_handle,
dev->coherent_dma_mask, flag,
dev_to_node(dev));
}
static void dma_nommu_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
struct iommu_table *iommu;
/* See comments in dma_nommu_alloc_coherent() */
if (dma_nommu_dma_supported(dev, dev->coherent_dma_mask))
return __dma_nommu_free_coherent(dev, size, vaddr, dma_handle,
attrs);
/* Maybe we used an iommu ... */
iommu = get_iommu_table_base(dev);
/* If we hit that we should have never allocated in the first
* place so how come we are freeing ?
*/
if (WARN_ON(!iommu))
return;
iommu_free_coherent(iommu, size, vaddr, dma_handle);
}
int dma_nommu_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t handle, size_t size,
unsigned long attrs)
{
unsigned long pfn;
#ifdef CONFIG_NOT_COHERENT_CACHE
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
pfn = __dma_get_coherent_pfn((unsigned long)cpu_addr);
#else
pfn = page_to_pfn(virt_to_page(cpu_addr));
#endif
return remap_pfn_range(vma, vma->vm_start,
pfn + vma->vm_pgoff,
vma->vm_end - vma->vm_start,
vma->vm_page_prot);
}
static int dma_nommu_map_sg(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction direction,
unsigned long attrs)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nents, i) {
sg->dma_address = sg_phys(sg) + get_dma_offset(dev);
sg->dma_length = sg->length;
if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
continue;
__dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
}
return nents;
}
static void dma_nommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction direction,
unsigned long attrs)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nents, i)
__dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
}
static u64 dma_nommu_get_required_mask(struct device *dev)
{
u64 end, mask;
end = memblock_end_of_DRAM() + get_dma_offset(dev);
mask = 1ULL << (fls64(end) - 1);
mask += mask - 1;
return mask;
}
static inline dma_addr_t dma_nommu_map_page(struct device *dev,
struct page *page,
unsigned long offset,
size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
__dma_sync_page(page, offset, size, dir);
return page_to_phys(page) + offset + get_dma_offset(dev);
}
static inline void dma_nommu_unmap_page(struct device *dev,
dma_addr_t dma_address,
size_t size,
enum dma_data_direction direction,
unsigned long attrs)
{
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
__dma_sync(bus_to_virt(dma_address), size, direction);
}
#ifdef CONFIG_NOT_COHERENT_CACHE
static inline void dma_nommu_sync_sg(struct device *dev,
struct scatterlist *sgl, int nents,
enum dma_data_direction direction)
{
struct scatterlist *sg;
int i;
for_each_sg(sgl, sg, nents, i)
__dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
}
static inline void dma_nommu_sync_single(struct device *dev,
dma_addr_t dma_handle, size_t size,
enum dma_data_direction direction)
{
__dma_sync(bus_to_virt(dma_handle), size, direction);
}
#endif
const struct dma_map_ops dma_nommu_ops = {
.alloc = dma_nommu_alloc_coherent,
.free = dma_nommu_free_coherent,
.mmap = dma_nommu_mmap_coherent,
.map_sg = dma_nommu_map_sg,
.unmap_sg = dma_nommu_unmap_sg,
.dma_supported = dma_nommu_dma_supported,
.map_page = dma_nommu_map_page,
.unmap_page = dma_nommu_unmap_page,
.get_required_mask = dma_nommu_get_required_mask,
#ifdef CONFIG_NOT_COHERENT_CACHE
.sync_single_for_cpu = dma_nommu_sync_single,
.sync_single_for_device = dma_nommu_sync_single,
.sync_sg_for_cpu = dma_nommu_sync_sg,
.sync_sg_for_device = dma_nommu_sync_sg,
#endif
};
EXPORT_SYMBOL(dma_nommu_ops);
int dma_set_coherent_mask(struct device *dev, u64 mask)
{
if (!dma_supported(dev, mask)) {
/*
* We need to special case the direct DMA ops which can
* support a fallback for coherent allocations. There
* is no dma_op->set_coherent_mask() so we have to do
* things the hard way:
*/
if (get_dma_ops(dev) != &dma_nommu_ops ||
get_iommu_table_base(dev) == NULL ||
!dma_iommu_dma_supported(dev, mask))
return -EIO;
}
dev->coherent_dma_mask = mask;
return 0;
}
EXPORT_SYMBOL(dma_set_coherent_mask);
int dma_set_mask(struct device *dev, u64 dma_mask)
{
if (ppc_md.dma_set_mask)
return ppc_md.dma_set_mask(dev, dma_mask);
if (dev_is_pci(dev)) {
struct pci_dev *pdev = to_pci_dev(dev);
struct pci_controller *phb = pci_bus_to_host(pdev->bus);
if (phb->controller_ops.dma_set_mask)
return phb->controller_ops.dma_set_mask(pdev, dma_mask);
}
if (!dev->dma_mask || !dma_supported(dev, dma_mask))
return -EIO;
*dev->dma_mask = dma_mask;
return 0;
}
EXPORT_SYMBOL(dma_set_mask);
u64 __dma_get_required_mask(struct device *dev)
{
const struct dma_map_ops *dma_ops = get_dma_ops(dev);
if (unlikely(dma_ops == NULL))
return 0;
if (dma_ops->get_required_mask)
return dma_ops->get_required_mask(dev);
return DMA_BIT_MASK(8 * sizeof(dma_addr_t));
}
u64 dma_get_required_mask(struct device *dev)
{
if (ppc_md.dma_get_required_mask)
return ppc_md.dma_get_required_mask(dev);
if (dev_is_pci(dev)) {
struct pci_dev *pdev = to_pci_dev(dev);
struct pci_controller *phb = pci_bus_to_host(pdev->bus);
if (phb->controller_ops.dma_get_required_mask)
return phb->controller_ops.dma_get_required_mask(pdev);
}
return __dma_get_required_mask(dev);
}
EXPORT_SYMBOL_GPL(dma_get_required_mask);
static int __init dma_init(void)
{
#ifdef CONFIG_IBMVIO
dma_debug_add_bus(&vio_bus_type);
#endif
return 0;
}
fs_initcall(dma_init);

17
arch/powerpc/kernel/dt_cpu_ftrs.c
View File

@ -666,8 +666,10 @@ static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f)
m = &dt_cpu_feature_match_table[i];
if (!strcmp(f->name, m->name)) {
known = true;
if (m->enable(f))
if (m->enable(f)) {
cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask;
break;
}
pr_info("not enabling: %s (disabled or unsupported by kernel)\n",
f->name);
@ -675,17 +677,12 @@ static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f)
}
}
if (!known && enable_unknown) {
if (!feat_try_enable_unknown(f)) {
pr_info("not enabling: %s (unknown and unsupported by kernel)\n",
f->name);
return false;
}
if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) {
pr_info("not enabling: %s (unknown and unsupported by kernel)\n",
f->name);
return false;
}
if (m->cpu_ftr_bit_mask)
cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask;
if (known)
pr_debug("enabling: %s\n", f->name);
else

194
arch/powerpc/kernel/eeh.c
View File

@ -109,7 +109,14 @@ EXPORT_SYMBOL(eeh_subsystem_flags);
* frozen count in last hour exceeds this limit, the PE will
* be forced to be offline permanently.
*/
int eeh_max_freezes = 5;
u32 eeh_max_freezes = 5;
/*
* Controls whether a recovery event should be scheduled when an
* isolated device is discovered. This is only really useful for
* debugging problems with the EEH core.
*/
bool eeh_debugfs_no_recover;
/* Platform dependent EEH operations */
struct eeh_ops *eeh_ops = NULL;
@ -823,15 +830,15 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat
switch (state) {
case pcie_deassert_reset:
eeh_ops->reset(pe, EEH_RESET_DEACTIVATE);
eeh_unfreeze_pe(pe, false);
eeh_unfreeze_pe(pe);
if (!(pe->type & EEH_PE_VF))
eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED);
eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true);
eeh_pe_dev_traverse(pe, eeh_restore_dev_state, dev);
eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true);
break;
case pcie_hot_reset:
eeh_pe_mark_isolated(pe);
eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED);
eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true);
eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);
eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev);
if (!(pe->type & EEH_PE_VF))
@ -840,7 +847,7 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat
break;
case pcie_warm_reset:
eeh_pe_mark_isolated(pe);
eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED);
eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true);
eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);
eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev);
if (!(pe->type & EEH_PE_VF))
@ -848,7 +855,7 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat
eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL);
break;
default:
eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED);
eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED, true);
return -EINVAL;
};
@ -877,6 +884,24 @@ static void *eeh_set_dev_freset(struct eeh_dev *edev, void *flag)
return NULL;
}
static void eeh_pe_refreeze_passed(struct eeh_pe *root)
{
struct eeh_pe *pe;
int state;
eeh_for_each_pe(root, pe) {
if (eeh_pe_passed(pe)) {
state = eeh_ops->get_state(pe, NULL);
if (state &
(EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED)) {
pr_info("EEH: Passed-through PE PHB#%x-PE#%x was thawed by reset, re-freezing for safety.\n",
pe->phb->global_number, pe->addr);
eeh_pe_set_option(pe, EEH_OPT_FREEZE_PE);
}
}
}
}
/**
* eeh_pe_reset_full - Complete a full reset process on the indicated PE
* @pe: EEH PE
@ -889,12 +914,12 @@ static void *eeh_set_dev_freset(struct eeh_dev *edev, void *flag)
*
* This function will attempt to reset a PE three times before failing.
*/
int eeh_pe_reset_full(struct eeh_pe *pe)
int eeh_pe_reset_full(struct eeh_pe *pe, bool include_passed)
{
int reset_state = (EEH_PE_RESET | EEH_PE_CFG_BLOCKED);
int type = EEH_RESET_HOT;
unsigned int freset = 0;
int i, state, ret;
int i, state = 0, ret;
/*
* Determine the type of reset to perform - hot or fundamental.
@ -911,32 +936,42 @@ int eeh_pe_reset_full(struct eeh_pe *pe)
/* Make three attempts at resetting the bus */
for (i = 0; i < 3; i++) {
ret = eeh_pe_reset(pe, type);
if (ret)
break;
ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE);
if (ret)
break;
ret = eeh_pe_reset(pe, type, include_passed);
if (!ret)
ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE,
include_passed);
if (ret) {
ret = -EIO;
pr_warn("EEH: Failure %d resetting PHB#%x-PE#%x (attempt %d)\n\n",
state, pe->phb->global_number, pe->addr, i + 1);
continue;
}
if (i)
pr_warn("EEH: PHB#%x-PE#%x: Successful reset (attempt %d)\n",
pe->phb->global_number, pe->addr, i + 1);
/* Wait until the PE is in a functioning state */
state = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC);
if (state < 0) {
pr_warn("%s: Unrecoverable slot failure on PHB#%x-PE#%x",
__func__, pe->phb->global_number, pe->addr);
pr_warn("EEH: Unrecoverable slot failure on PHB#%x-PE#%x",
pe->phb->global_number, pe->addr);
ret = -ENOTRECOVERABLE;
break;
}
if (eeh_state_active(state))
break;
/* Set error in case this is our last attempt */
ret = -EIO;
pr_warn("%s: Failure %d resetting PHB#%x-PE#%x\n (%d)\n",
__func__, state, pe->phb->global_number, pe->addr, (i + 1));
else
pr_warn("EEH: PHB#%x-PE#%x: Slot inactive after reset: 0x%x (attempt %d)\n",
pe->phb->global_number, pe->addr, state, i + 1);
}
eeh_pe_state_clear(pe, reset_state);
/* Resetting the PE may have unfrozen child PEs. If those PEs have been
* (potentially) passed through to a guest, re-freeze them:
*/
if (!include_passed)
eeh_pe_refreeze_passed(pe);
eeh_pe_state_clear(pe, reset_state, true);
return ret;
}
@ -1309,7 +1344,7 @@ void eeh_remove_device(struct pci_dev *dev)
edev->mode &= ~EEH_DEV_SYSFS;
}
int eeh_unfreeze_pe(struct eeh_pe *pe, bool sw_state)
int eeh_unfreeze_pe(struct eeh_pe *pe)
{
int ret;
@ -1327,10 +1362,6 @@ int eeh_unfreeze_pe(struct eeh_pe *pe, bool sw_state)
return ret;
}
/* Clear software isolated state */
if (sw_state && (pe->state & EEH_PE_ISOLATED))
eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
return ret;
}
@ -1382,7 +1413,10 @@ static int eeh_pe_change_owner(struct eeh_pe *pe)
}
}
return eeh_unfreeze_pe(pe, true);
ret = eeh_unfreeze_pe(pe);
if (!ret)
eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true);
return ret;
}
/**
@ -1612,13 +1646,12 @@ int eeh_pe_get_state(struct eeh_pe *pe)
}
EXPORT_SYMBOL_GPL(eeh_pe_get_state);
static int eeh_pe_reenable_devices(struct eeh_pe *pe)
static int eeh_pe_reenable_devices(struct eeh_pe *pe, bool include_passed)
{
struct eeh_dev *edev, *tmp;
struct pci_dev *pdev;
int ret = 0;
/* Restore config space */
eeh_pe_restore_bars(pe);
/*
@ -1639,7 +1672,14 @@ static int eeh_pe_reenable_devices(struct eeh_pe *pe)
}
/* The PE is still in frozen state */
return eeh_unfreeze_pe(pe, true);
if (include_passed || !eeh_pe_passed(pe)) {
ret = eeh_unfreeze_pe(pe);
} else
pr_info("EEH: Note: Leaving passthrough PHB#%x-PE#%x frozen.\n",
pe->phb->global_number, pe->addr);
if (!ret)
eeh_pe_state_clear(pe, EEH_PE_ISOLATED, include_passed);
return ret;
}
@ -1652,7 +1692,7 @@ static int eeh_pe_reenable_devices(struct eeh_pe *pe)
* indicated type, either fundamental reset or hot reset.
* PE reset is the most important part for error recovery.
*/
int eeh_pe_reset(struct eeh_pe *pe, int option)
int eeh_pe_reset(struct eeh_pe *pe, int option, bool include_passed)
{
int ret = 0;
@ -1666,11 +1706,11 @@ int eeh_pe_reset(struct eeh_pe *pe, int option)
switch (option) {
case EEH_RESET_DEACTIVATE:
ret = eeh_ops->reset(pe, option);
eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED);
eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, include_passed);
if (ret)
break;
ret = eeh_pe_reenable_devices(pe);
ret = eeh_pe_reenable_devices(pe, include_passed);
break;
case EEH_RESET_HOT:
case EEH_RESET_FUNDAMENTAL:
@ -1796,22 +1836,64 @@ static int eeh_enable_dbgfs_get(void *data, u64 *val)
return 0;
}
static int eeh_freeze_dbgfs_set(void *data, u64 val)
{
eeh_max_freezes = val;
return 0;
}
static int eeh_freeze_dbgfs_get(void *data, u64 *val)
{
*val = eeh_max_freezes;
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get,
eeh_enable_dbgfs_set, "0x%llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(eeh_freeze_dbgfs_ops, eeh_freeze_dbgfs_get,
eeh_freeze_dbgfs_set, "0x%llx\n");
static ssize_t eeh_force_recover_write(struct file *filp,
const char __user *user_buf,
size_t count, loff_t *ppos)
{
struct pci_controller *hose;
uint32_t phbid, pe_no;
struct eeh_pe *pe;
char buf[20];
int ret;
ret = simple_write_to_buffer(buf, sizeof(buf), ppos, user_buf, count);
if (!ret)
return -EFAULT;
/*
* When PE is NULL the event is a "special" event. Rather than
* recovering a specific PE it forces the EEH core to scan for failed
* PHBs and recovers each. This needs to be done before any device
* recoveries can occur.
*/
if (!strncmp(buf, "hwcheck", 7)) {
__eeh_send_failure_event(NULL);
return count;
}
ret = sscanf(buf, "%x:%x", &phbid, &pe_no);
if (ret != 2)
return -EINVAL;
hose = pci_find_controller_for_domain(phbid);
if (!hose)
return -ENODEV;
/* Retrieve PE */
pe = eeh_pe_get(hose, pe_no, 0);
if (!pe)
return -ENODEV;
/*
* We don't do any state checking here since the detection
* process is async to the recovery process. The recovery
* thread *should* not break even if we schedule a recovery
* from an odd state (e.g. PE removed, or recovery of a
* non-isolated PE)
*/
__eeh_send_failure_event(pe);
return ret < 0 ? ret : count;
}
static const struct file_operations eeh_force_recover_fops = {
.open = simple_open,
.llseek = no_llseek,
.write = eeh_force_recover_write,
};
#endif
static int __init eeh_init_proc(void)
@ -1822,9 +1904,15 @@ static int __init eeh_init_proc(void)
debugfs_create_file_unsafe("eeh_enable", 0600,
powerpc_debugfs_root, NULL,
&eeh_enable_dbgfs_ops);
debugfs_create_file_unsafe("eeh_max_freezes", 0600,
powerpc_debugfs_root, NULL,
&eeh_freeze_dbgfs_ops);
debugfs_create_u32("eeh_max_freezes", 0600,
powerpc_debugfs_root, &eeh_max_freezes);
debugfs_create_bool("eeh_disable_recovery", 0600,
powerpc_debugfs_root,
&eeh_debugfs_no_recover);
debugfs_create_file_unsafe("eeh_force_recover", 0600,
powerpc_debugfs_root, NULL,
&eeh_force_recover_fops);
eeh_cache_debugfs_init();
#endif
}

38
arch/powerpc/kernel/eeh_cache.c
View File

@ -26,6 +26,7 @@
#include <linux/spinlock.h>
#include <linux/atomic.h>
#include <asm/pci-bridge.h>
#include <asm/debugfs.h>
#include <asm/ppc-pci.h>
@ -113,7 +114,7 @@ static void eeh_addr_cache_print(struct pci_io_addr_cache *cache)
while (n) {
struct pci_io_addr_range *piar;
piar = rb_entry(n, struct pci_io_addr_range, rb_node);
pr_debug("PCI: %s addr range %d [%pap-%pap]: %s\n",
pr_info("PCI: %s addr range %d [%pap-%pap]: %s\n",
(piar->flags & IORESOURCE_IO) ? "i/o" : "mem", cnt,
&piar->addr_lo, &piar->addr_hi, pci_name(piar->pcidev));
cnt++;
@ -157,10 +158,8 @@ eeh_addr_cache_insert(struct pci_dev *dev, resource_size_t alo,
piar->pcidev = dev;
piar->flags = flags;
#ifdef DEBUG
pr_debug("PIAR: insert range=[%pap:%pap] dev=%s\n",
&alo, &ahi, pci_name(dev));
#endif
rb_link_node(&piar->rb_node, parent, p);
rb_insert_color(&piar->rb_node, &pci_io_addr_cache_root.rb_root);
@ -240,6 +239,8 @@ static inline void __eeh_addr_cache_rmv_dev(struct pci_dev *dev)
piar = rb_entry(n, struct pci_io_addr_range, rb_node);
if (piar->pcidev == dev) {
pr_debug("PIAR: remove range=[%pap:%pap] dev=%s\n",
&piar->addr_lo, &piar->addr_hi, pci_name(dev));
rb_erase(n, &pci_io_addr_cache_root.rb_root);
kfree(piar);
goto restart;
@ -298,9 +299,30 @@ void eeh_addr_cache_build(void)
eeh_addr_cache_insert_dev(dev);
eeh_sysfs_add_device(dev);
}
#ifdef DEBUG
/* Verify tree built up above, echo back the list of addrs. */
eeh_addr_cache_print(&pci_io_addr_cache_root);
#endif
}
static int eeh_addr_cache_show(struct seq_file *s, void *v)
{
struct pci_io_addr_range *piar;
struct rb_node *n;
spin_lock(&pci_io_addr_cache_root.piar_lock);
for (n = rb_first(&pci_io_addr_cache_root.rb_root); n; n = rb_next(n)) {
piar = rb_entry(n, struct pci_io_addr_range, rb_node);
seq_printf(s, "%s addr range [%pap-%pap]: %s\n",
(piar->flags & IORESOURCE_IO) ? "i/o" : "mem",
&piar->addr_lo, &piar->addr_hi, pci_name(piar->pcidev));
}
spin_unlock(&pci_io_addr_cache_root.piar_lock);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(eeh_addr_cache);
void eeh_cache_debugfs_init(void)
{
debugfs_create_file_unsafe("eeh_address_cache", 0400,
powerpc_debugfs_root, NULL,
&eeh_addr_cache_fops);
}

86
arch/powerpc/kernel/eeh_driver.c
View File

@ -510,22 +510,11 @@ static void *eeh_rmv_device(struct eeh_dev *edev, void *userdata)
* support EEH. So we just care about PCI devices for
* simplicity here.
*/
if (!dev || (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE))
return NULL;
/*
* We rely on count-based pcibios_release_device() to
* detach permanently offlined PEs. Unfortunately, that's
* not reliable enough. We might have the permanently
* offlined PEs attached, but we needn't take care of
* them and their child devices.
*/
if (eeh_dev_removed(edev))
if (!eeh_edev_actionable(edev) ||
(dev->hdr_type == PCI_HEADER_TYPE_BRIDGE))
return NULL;
if (rmv_data) {
if (eeh_pe_passed(edev->pe))
return NULL;
driver = eeh_pcid_get(dev);
if (driver) {
if (driver->err_handler &&
@ -539,8 +528,8 @@ static void *eeh_rmv_device(struct eeh_dev *edev, void *userdata)
}
/* Remove it from PCI subsystem */
pr_debug("EEH: Removing %s without EEH sensitive driver\n",
pci_name(dev));
pr_info("EEH: Removing %s without EEH sensitive driver\n",
pci_name(dev));
edev->mode |= EEH_DEV_DISCONNECTED;
if (rmv_data)
rmv_data->removed_dev_count++;
@ -591,34 +580,22 @@ static void *eeh_pe_detach_dev(struct eeh_pe *pe, void *userdata)
* PE reset (for 3 times), we try to clear the frozen state
* for 3 times as well.
*/
static void *__eeh_clear_pe_frozen_state(struct eeh_pe *pe, void *flag)
static int eeh_clear_pe_frozen_state(struct eeh_pe *root, bool include_passed)
{
bool clear_sw_state = *(bool *)flag;
int i, rc = 1;
struct eeh_pe *pe;
int i;
for (i = 0; rc && i < 3; i++)
rc = eeh_unfreeze_pe(pe, clear_sw_state);
/* Stop immediately on any errors */
if (rc) {
pr_warn("%s: Failure %d unfreezing PHB#%x-PE#%x\n",
__func__, rc, pe->phb->global_number, pe->addr);
return (void *)pe;
eeh_for_each_pe(root, pe) {
if (include_passed || !eeh_pe_passed(pe)) {
for (i = 0; i < 3; i++)
if (!eeh_unfreeze_pe(pe))
break;
if (i >= 3)
return -EIO;
}
}
return NULL;
}
static int eeh_clear_pe_frozen_state(struct eeh_pe *pe,
bool clear_sw_state)
{
void *rc;
rc = eeh_pe_traverse(pe, __eeh_clear_pe_frozen_state, &clear_sw_state);
if (!rc)
eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
return rc ? -EIO : 0;
eeh_pe_state_clear(root, EEH_PE_ISOLATED, include_passed);
return 0;
}
int eeh_pe_reset_and_recover(struct eeh_pe *pe)
@ -636,16 +613,16 @@ int eeh_pe_reset_and_recover(struct eeh_pe *pe)
eeh_pe_dev_traverse(pe, eeh_dev_save_state, NULL);
/* Issue reset */
ret = eeh_pe_reset_full(pe);
ret = eeh_pe_reset_full(pe, true);
if (ret) {
eeh_pe_state_clear(pe, EEH_PE_RECOVERING);
eeh_pe_state_clear(pe, EEH_PE_RECOVERING, true);
return ret;
}
/* Unfreeze the PE */
ret = eeh_clear_pe_frozen_state(pe, true);
if (ret) {
eeh_pe_state_clear(pe, EEH_PE_RECOVERING);
eeh_pe_state_clear(pe, EEH_PE_RECOVERING, true);
return ret;
}
@ -653,7 +630,7 @@ int eeh_pe_reset_and_recover(struct eeh_pe *pe)
eeh_pe_dev_traverse(pe, eeh_dev_restore_state, NULL);
/* Clear recovery mode */
eeh_pe_state_clear(pe, EEH_PE_RECOVERING);
eeh_pe_state_clear(pe, EEH_PE_RECOVERING, true);
return 0;
}
@ -676,6 +653,11 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus,
time64_t tstamp;
int cnt, rc;
struct eeh_dev *edev;
struct eeh_pe *tmp_pe;
bool any_passed = false;
eeh_for_each_pe(pe, tmp_pe)
any_passed |= eeh_pe_passed(tmp_pe);
/* pcibios will clear the counter; save the value */
cnt = pe->freeze_count;
@ -688,7 +670,7 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus,
* into pci_hp_add_devices().
*/
eeh_pe_state_mark(pe, EEH_PE_KEEP);
if (driver_eeh_aware || (pe->type & EEH_PE_VF)) {
if (any_passed || driver_eeh_aware || (pe->type & EEH_PE_VF)) {
eeh_pe_dev_traverse(pe, eeh_rmv_device, rmv_data);
} else {
pci_lock_rescan_remove();
@ -705,7 +687,7 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus,
* config accesses. So we prefer to block them. However, controlled
* PCI config accesses initiated from EEH itself are allowed.
*/
rc = eeh_pe_reset_full(pe);
rc = eeh_pe_reset_full(pe, false);
if (rc)
return rc;
@ -744,11 +726,11 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus,
eeh_add_virt_device(edev);
} else {
if (!driver_eeh_aware)
eeh_pe_state_clear(pe, EEH_PE_PRI_BUS);
eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true);
pci_hp_add_devices(bus);
}
}
eeh_pe_state_clear(pe, EEH_PE_KEEP);
eeh_pe_state_clear(pe, EEH_PE_KEEP, true);
pe->tstamp = tstamp;
pe->freeze_count = cnt;
@ -900,7 +882,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe)
* is still in frozen state. Clear it before
* resuming the PE.
*/
eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true);
result = PCI_ERS_RESULT_RECOVERED;
}
}
@ -977,7 +959,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe)
eeh_pe_dev_traverse(pe, eeh_rmv_device, NULL);
eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED);
} else {
eeh_pe_state_clear(pe, EEH_PE_PRI_BUS);
eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true);
eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED);
pci_lock_rescan_remove();
@ -987,7 +969,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe)
return;
}
}
eeh_pe_state_clear(pe, EEH_PE_RECOVERING);
eeh_pe_state_clear(pe, EEH_PE_RECOVERING, true);
}
/**
@ -1069,7 +1051,7 @@ void eeh_handle_special_event(void)
continue;
/* Notify all devices to be down */
eeh_pe_state_clear(pe, EEH_PE_PRI_BUS);
eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true);
eeh_set_channel_state(pe, pci_channel_io_perm_failure);
eeh_pe_report(
"error_detected(permanent failure)", pe,

16
arch/powerpc/kernel/eeh_event.c
View File

@ -121,7 +121,7 @@ int eeh_event_init(void)
* the actual event will be delivered in a normal context
* (from a workqueue).
*/
int eeh_send_failure_event(struct eeh_pe *pe)
int __eeh_send_failure_event(struct eeh_pe *pe)
{
unsigned long flags;
struct eeh_event *event;
@ -144,6 +144,20 @@ int eeh_send_failure_event(struct eeh_pe *pe)
return 0;
}
int eeh_send_failure_event(struct eeh_pe *pe)
{
/*
* If we've manually supressed recovery events via debugfs
* then just drop it on the floor.
*/
if (eeh_debugfs_no_recover) {
pr_err("EEH: Event dropped due to no_recover setting\n");
return 0;
}
return __eeh_send_failure_event(pe);
}
/**
* eeh_remove_event - Remove EEH event from the queue
* @pe: Event binding to the PE

76
arch/powerpc/kernel/eeh_pe.c
View File

@ -657,62 +657,52 @@ void eeh_pe_dev_mode_mark(struct eeh_pe *pe, int mode)
}
/**
* __eeh_pe_state_clear - Clear state for the PE
* eeh_pe_state_clear - Clear state for the PE
* @data: EEH PE
* @flag: state
* @state: state
* @include_passed: include passed-through devices?
*
* The function is used to clear the indicated state from the
* given PE. Besides, we also clear the check count of the PE
* as well.
*/
static void *__eeh_pe_state_clear(struct eeh_pe *pe, void *flag)
void eeh_pe_state_clear(struct eeh_pe *root, int state, bool include_passed)
{
int state = *((int *)flag);
struct eeh_pe *pe;
struct eeh_dev *edev, *tmp;
struct pci_dev *pdev;
/* Keep the state of permanently removed PE intact */
if (pe->state & EEH_PE_REMOVED)
return NULL;
pe->state &= ~state;
/*
* Special treatment on clearing isolated state. Clear
* check count since last isolation and put all affected
* devices to normal state.
*/
if (!(state & EEH_PE_ISOLATED))
return NULL;
pe->check_count = 0;
eeh_pe_for_each_dev(pe, edev, tmp) {
pdev = eeh_dev_to_pci_dev(edev);
if (!pdev)
eeh_for_each_pe(root, pe) {
/* Keep the state of permanently removed PE intact */
if (pe->state & EEH_PE_REMOVED)
continue;
pdev->error_state = pci_channel_io_normal;
if (!include_passed && eeh_pe_passed(pe))
continue;
pe->state &= ~state;
/*
* Special treatment on clearing isolated state. Clear
* check count since last isolation and put all affected
* devices to normal state.
*/
if (!(state & EEH_PE_ISOLATED))
continue;
pe->check_count = 0;
eeh_pe_for_each_dev(pe, edev, tmp) {
pdev = eeh_dev_to_pci_dev(edev);
if (!pdev)
continue;
pdev->error_state = pci_channel_io_normal;
}
/* Unblock PCI config access if required */
if (pe->state & EEH_PE_CFG_RESTRICTED)
pe->state &= ~EEH_PE_CFG_BLOCKED;
}
/* Unblock PCI config access if required */
if (pe->state & EEH_PE_CFG_RESTRICTED)
pe->state &= ~EEH_PE_CFG_BLOCKED;
return NULL;
}
/**
* eeh_pe_state_clear - Clear state for the PE and its children
* @pe: PE
* @state: state to be cleared
*
* When the PE and its children has been recovered from error,
* we need clear the error state for that. The function is used
* for the purpose.
*/
void eeh_pe_state_clear(struct eeh_pe *pe, int state)
{
eeh_pe_traverse(pe, __eeh_pe_state_clear, &state);
}
/*

3
arch/powerpc/kernel/eeh_sysfs.c
View File

@ -82,8 +82,9 @@ static ssize_t eeh_pe_state_store(struct device *dev,
if (!(edev->pe->state & EEH_PE_ISOLATED))
return count;
if (eeh_unfreeze_pe(edev->pe, true))
if (eeh_unfreeze_pe(edev->pe))
return -EIO;
eeh_pe_state_clear(edev->pe, EEH_PE_ISOLATED, true);
return count;
}

97
arch/powerpc/kernel/entry_32.S
View File

@ -97,14 +97,11 @@ crit_transfer_to_handler:
mfspr r0,SPRN_SRR1
stw r0,_SRR1(r11)
/* set the stack limit to the current stack
* and set the limit to protect the thread_info
* struct
*/
/* set the stack limit to the current stack */
mfspr r8,SPRN_SPRG_THREAD
lwz r0,KSP_LIMIT(r8)
stw r0,SAVED_KSP_LIMIT(r11)
rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
stw r0,KSP_LIMIT(r8)
/* fall through */
#endif
@ -121,14 +118,11 @@ crit_transfer_to_handler:
mfspr r0,SPRN_SRR1
stw r0,crit_srr1@l(0)
/* set the stack limit to the current stack
* and set the limit to protect the thread_info
* struct
*/
/* set the stack limit to the current stack */
mfspr r8,SPRN_SPRG_THREAD
lwz r0,KSP_LIMIT(r8)
stw r0,saved_ksp_limit@l(0)
rlwimi r0,r1,0,0,(31-THREAD_SHIFT)
rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
stw r0,KSP_LIMIT(r8)
/* fall through */
#endif
@ -157,7 +151,6 @@ transfer_to_handler:
stw r2,_XER(r11)
mfspr r12,SPRN_SPRG_THREAD
addi r2,r12,-THREAD
tovirt(r2,r2) /* set r2 to current */
beq 2f /* if from user, fix up THREAD.regs */
addi r11,r1,STACK_FRAME_OVERHEAD
stw r11,PT_REGS(r12)
@ -166,6 +159,9 @@ transfer_to_handler:
internal debug mode bit to do this. */
lwz r12,THREAD_DBCR0(r12)
andis. r12,r12,DBCR0_IDM@h
#endif
ACCOUNT_CPU_USER_ENTRY(r2, r11, r12)
#if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
beq+ 3f
/* From user and task is ptraced - load up global dbcr0 */
li r12,-1 /* clear all pending debug events */
@ -174,8 +170,7 @@ transfer_to_handler:
tophys(r11,r11)
addi r11,r11,global_dbcr0@l
#ifdef CONFIG_SMP
CURRENT_THREAD_INFO(r9, r1)
lwz r9,TI_CPU(r9)
lwz r9,TASK_CPU(r2)
slwi r9,r9,3
add r11,r11,r9
#endif
@ -185,11 +180,6 @@ transfer_to_handler:
addi r12,r12,-1
stw r12,4(r11)
#endif
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
CURRENT_THREAD_INFO(r9, r1)
tophys(r9, r9)
ACCOUNT_CPU_USER_ENTRY(r9, r11, r12)
#endif
b 3f
@ -201,9 +191,7 @@ transfer_to_handler:
ble- stack_ovf /* then the kernel stack overflowed */
5:
#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
CURRENT_THREAD_INFO(r9, r1)
tophys(r9,r9) /* check local flags */
lwz r12,TI_LOCAL_FLAGS(r9)
lwz r12,TI_LOCAL_FLAGS(r2)
mtcrf 0x01,r12
bt- 31-TLF_NAPPING,4f
bt- 31-TLF_SLEEPING,7f
@ -212,6 +200,7 @@ transfer_to_handler:
transfer_to_handler_cont:
3:
mflr r9
tovirt(r2, r2) /* set r2 to current */
lwz r11,0(r9) /* virtual address of handler */
lwz r9,4(r9) /* where to go when done */
#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
@ -275,11 +264,11 @@ reenable_mmu: /* re-enable mmu so we can */
#if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
4: rlwinm r12,r12,0,~_TLF_NAPPING
stw r12,TI_LOCAL_FLAGS(r9)
stw r12,TI_LOCAL_FLAGS(r2)
b power_save_ppc32_restore
7: rlwinm r12,r12,0,~_TLF_SLEEPING
stw r12,TI_LOCAL_FLAGS(r9)
stw r12,TI_LOCAL_FLAGS(r2)
lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
rlwinm r9,r9,0,~MSR_EE
lwz r12,_LINK(r11) /* and return to address in LR */
@ -351,8 +340,7 @@ _GLOBAL(DoSyscall)
mtmsr r11
1:
#endif /* CONFIG_TRACE_IRQFLAGS */
CURRENT_THREAD_INFO(r10, r1)
lwz r11,TI_FLAGS(r10)
lwz r11,TI_FLAGS(r2)
andi. r11,r11,_TIF_SYSCALL_DOTRACE
bne- syscall_dotrace
syscall_dotrace_cont:
@ -385,13 +373,12 @@ ret_from_syscall:
lwz r3,GPR3(r1)
#endif
mr r6,r3
CURRENT_THREAD_INFO(r12, r1)
/* disable interrupts so current_thread_info()->flags can't change */
LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
/* Note: We don't bother telling lockdep about it */
SYNC
MTMSRD(r10)
lwz r9,TI_FLAGS(r12)
lwz r9,TI_FLAGS(r2)
li r8,-MAX_ERRNO
andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
bne- syscall_exit_work
@ -438,8 +425,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
andi. r4,r8,MSR_PR
beq 3f
CURRENT_THREAD_INFO(r4, r1)
ACCOUNT_CPU_USER_EXIT(r4, r5, r7)
ACCOUNT_CPU_USER_EXIT(r2, r5, r7)
3:
#endif
lwz r4,_LINK(r1)
@ -532,7 +518,7 @@ syscall_exit_work:
/* Clear per-syscall TIF flags if any are set. */
li r11,_TIF_PERSYSCALL_MASK
addi r12,r12,TI_FLAGS
addi r12,r2,TI_FLAGS
3: lwarx r8,0,r12
andc r8,r8,r11
#ifdef CONFIG_IBM405_ERR77
@ -540,7 +526,6 @@ syscall_exit_work:
#endif
stwcx. r8,0,r12
bne- 3b
subi r12,r12,TI_FLAGS
4: /* Anything which requires enabling interrupts? */
andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
@ -745,6 +730,9 @@ fast_exception_return:
mtcr r10
lwz r10,_LINK(r11)
mtlr r10
/* Clear the exception_marker on the stack to avoid confusing stacktrace */
li r10, 0
stw r10, 8(r11)
REST_GPR(10, r11)
#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
mtspr SPRN_NRI, r0
@ -819,8 +807,7 @@ ret_from_except:
user_exc_return: /* r10 contains MSR_KERNEL here */
/* Check current_thread_info()->flags */
CURRENT_THREAD_INFO(r9, r1)
lwz r9,TI_FLAGS(r9)
lwz r9,TI_FLAGS(r2)
andi. r0,r9,_TIF_USER_WORK_MASK
bne do_work
@ -832,18 +819,14 @@ restore_user:
andis. r10,r0,DBCR0_IDM@h
bnel- load_dbcr0
#endif
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
CURRENT_THREAD_INFO(r9, r1)
ACCOUNT_CPU_USER_EXIT(r9, r10, r11)
#endif
ACCOUNT_CPU_USER_EXIT(r2, r10, r11)
b restore
/* N.B. the only way to get here is from the beq following ret_from_except. */
resume_kernel:
/* check current_thread_info, _TIF_EMULATE_STACK_STORE */
CURRENT_THREAD_INFO(r9, r1)
lwz r8,TI_FLAGS(r9)
lwz r8,TI_FLAGS(r2)
andis. r0,r8,_TIF_EMULATE_STACK_STORE@h
beq+ 1f
@ -869,7 +852,7 @@ resume_kernel:
/* Clear _TIF_EMULATE_STACK_STORE flag */
lis r11,_TIF_EMULATE_STACK_STORE@h
addi r5,r9,TI_FLAGS
addi r5,r2,TI_FLAGS
0: lwarx r8,0,r5
andc r8,r8,r11
#ifdef CONFIG_IBM405_ERR77
@ -881,7 +864,7 @@ resume_kernel:
#ifdef CONFIG_PREEMPT
/* check current_thread_info->preempt_count */
lwz r0,TI_PREEMPT(r9)
lwz r0,TI_PREEMPT(r2)
cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
bne restore
andi. r8,r8,_TIF_NEED_RESCHED
@ -897,8 +880,7 @@ resume_kernel:
bl trace_hardirqs_off
#endif
1: bl preempt_schedule_irq
CURRENT_THREAD_INFO(r9, r1)
lwz r3,TI_FLAGS(r9)
lwz r3,TI_FLAGS(r2)
andi. r0,r3,_TIF_NEED_RESCHED
bne- 1b
#ifdef CONFIG_TRACE_IRQFLAGS
@ -982,6 +964,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
mtcrf 0xFF,r10
mtlr r11
/* Clear the exception_marker on the stack to avoid confusing stacktrace */
li r10, 0
stw r10, 8(r1)
/*
* Once we put values in SRR0 and SRR1, we are in a state
* where exceptions are not recoverable, since taking an
@ -997,9 +982,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
.globl exc_exit_restart
exc_exit_restart:
lwz r12,_NIP(r1)
#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
mtspr SPRN_NRI, r0
#endif
mtspr SPRN_SRR0,r12
mtspr SPRN_SRR1,r9
REST_4GPRS(9, r1)
@ -1021,6 +1003,9 @@ exc_exit_restart_end:
mtlr r11
lwz r10,_CCR(r1)
mtcrf 0xff,r10
/* Clear the exception_marker on the stack to avoid confusing stacktrace */
li r10, 0
stw r10, 8(r1)
REST_2GPRS(9, r1)
.globl exc_exit_restart
exc_exit_restart:
@ -1166,10 +1151,6 @@ ret_from_debug_exc:
mfspr r9,SPRN_SPRG_THREAD
lwz r10,SAVED_KSP_LIMIT(r1)
stw r10,KSP_LIMIT(r9)
lwz r9,THREAD_INFO-THREAD(r9)
CURRENT_THREAD_INFO(r10, r1)
lwz r10,TI_PREEMPT(r10)
stw r10,TI_PREEMPT(r9)
RESTORE_xSRR(SRR0,SRR1);
RESTORE_xSRR(CSRR0,CSRR1);
RESTORE_MMU_REGS;
@ -1201,8 +1182,7 @@ load_dbcr0:
lis r11,global_dbcr0@ha
addi r11,r11,global_dbcr0@l
#ifdef CONFIG_SMP
CURRENT_THREAD_INFO(r9, r1)
lwz r9,TI_CPU(r9)
lwz r9,TASK_CPU(r2)
slwi r9,r9,3
add r11,r11,r9
#endif
@ -1242,8 +1222,7 @@ recheck:
LOAD_MSR_KERNEL(r10,MSR_KERNEL)
SYNC
MTMSRD(r10) /* disable interrupts */
CURRENT_THREAD_INFO(r9, r1)
lwz r9,TI_FLAGS(r9)
lwz r9,TI_FLAGS(r2)
andi. r0,r9,_TIF_NEED_RESCHED
bne- do_resched
andi. r0,r9,_TIF_USER_WORK_MASK
@ -1292,10 +1271,13 @@ BEGIN_FTR_SECTION
END_FTR_SECTION_IFSET(CPU_FTR_601)
lwz r3,_TRAP(r1)
andi. r0,r3,1
beq 4f
beq 5f
SAVE_NVGPRS(r1)
rlwinm r3,r3,0,0,30
stw r3,_TRAP(r1)
5: mfspr r2,SPRN_SPRG_THREAD
addi r2,r2,-THREAD
tovirt(r2,r2) /* set back r2 to current */
4: addi r3,r1,STACK_FRAME_OVERHEAD
bl unrecoverable_exception
/* shouldn't return */
@ -1335,7 +1317,7 @@ _GLOBAL(enter_rtas)
MTMSRD(r0) /* don't get trashed */
li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
mtlr r6
mtspr SPRN_SPRG_RTAS,r7
stw r7, THREAD + RTAS_SP(r2)
mtspr SPRN_SRR0,r8
mtspr SPRN_SRR1,r9
RFI
@ -1344,7 +1326,8 @@ _GLOBAL(enter_rtas)
lwz r9,8(r9) /* original msr value */
addi r1,r1,INT_FRAME_SIZE
li r0,0
mtspr SPRN_SPRG_RTAS,r0
tophys(r7, r2)
stw r0, THREAD + RTAS_SP(r7)
mtspr SPRN_SRR0,r8
mtspr SPRN_SRR1,r9
RFI /* return to caller */

53
arch/powerpc/kernel/entry_64.S
View File

@ -166,7 +166,7 @@ system_call: /* label this so stack traces look sane */
li r10,IRQS_ENABLED
std r10,SOFTE(r1)
CURRENT_THREAD_INFO(r11, r1)
ld r11, PACA_THREAD_INFO(r13)
ld r10,TI_FLAGS(r11)
andi. r11,r10,_TIF_SYSCALL_DOTRACE
bne .Lsyscall_dotrace /* does not return */
@ -213,7 +213,7 @@ system_call: /* label this so stack traces look sane */
ld r3,RESULT(r1)
#endif
CURRENT_THREAD_INFO(r12, r1)
ld r12, PACA_THREAD_INFO(r13)
ld r8,_MSR(r1)
#ifdef CONFIG_PPC_BOOK3S
@ -236,18 +236,14 @@ system_call_exit:
/*
* Disable interrupts so current_thread_info()->flags can't change,
* and so that we don't get interrupted after loading SRR0/1.
*
* Leave MSR_RI enabled for now, because with THREAD_INFO_IN_TASK we
* could fault on the load of the TI_FLAGS below.
*/
#ifdef CONFIG_PPC_BOOK3E
wrteei 0
#else
/*
* For performance reasons we clear RI the same time that we
* clear EE. We only need to clear RI just before we restore r13
* below, but batching it with EE saves us one expensive mtmsrd call.
* We have to be careful to restore RI if we branch anywhere from
* here (eg syscall_exit_work).
*/
li r11,0
li r11,MSR_RI
mtmsrd r11,1
#endif /* CONFIG_PPC_BOOK3E */
@ -263,15 +259,7 @@ system_call_exit:
bne 3f
#endif
2: addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_PPC_BOOK3S
li r10,MSR_RI
mtmsrd r10,1 /* Restore RI */
#endif
bl restore_math
#ifdef CONFIG_PPC_BOOK3S
li r11,0
mtmsrd r11,1
#endif
ld r8,_MSR(r1)
ld r3,RESULT(r1)
li r11,-MAX_ERRNO
@ -287,6 +275,16 @@ END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
andi. r6,r8,MSR_PR
ld r4,_LINK(r1)
#ifdef CONFIG_PPC_BOOK3S
/*
* Clear MSR_RI, MSR_EE is already and remains disabled. We could do
* this later, but testing shows that doing it here causes less slow
* down than doing it closer to the rfid.
*/
li r11,0
mtmsrd r11,1
#endif
beq- 1f
ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
@ -348,7 +346,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
/* Repopulate r9 and r10 for the syscall path */
addi r9,r1,STACK_FRAME_OVERHEAD
CURRENT_THREAD_INFO(r10, r1)
ld r10, PACA_THREAD_INFO(r13)
ld r10,TI_FLAGS(r10)
cmpldi r0,NR_syscalls
@ -363,10 +361,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
b .Lsyscall_exit
.Lsyscall_exit_work:
#ifdef CONFIG_PPC_BOOK3S
li r10,MSR_RI
mtmsrd r10,1 /* Restore RI */
#endif
/* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
If TIF_NOERROR is set, just save r3 as it is. */
@ -695,7 +689,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
2:
#endif /* CONFIG_PPC_BOOK3S_64 */
CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
clrrdi r7, r8, THREAD_SHIFT /* base of new stack */
/* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
because we don't need to leave the 288-byte ABI gap at the
top of the kernel stack. */
@ -746,7 +740,7 @@ _GLOBAL(ret_from_except_lite)
mtmsrd r10,1 /* Update machine state */
#endif /* CONFIG_PPC_BOOK3E */
CURRENT_THREAD_INFO(r9, r1)
ld r9, PACA_THREAD_INFO(r13)
ld r3,_MSR(r1)
#ifdef CONFIG_PPC_BOOK3E
ld r10,PACACURRENT(r13)
@ -860,7 +854,7 @@ resume_kernel:
1: bl preempt_schedule_irq
/* Re-test flags and eventually loop */
CURRENT_THREAD_INFO(r9, r1)
ld r9, PACA_THREAD_INFO(r13)
ld r4,TI_FLAGS(r9)
andi. r0,r4,_TIF_NEED_RESCHED
bne 1b
@ -1002,6 +996,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
ld r2,_NIP(r1)
mtspr SPRN_SRR0,r2
/*
* Leaving a stale exception_marker on the stack can confuse
* the reliable stack unwinder later on. Clear it.
*/
li r2,0
std r2,STACK_FRAME_OVERHEAD-16(r1)
ld r0,GPR0(r1)
ld r2,GPR2(r1)
ld r3,GPR3(r1)

5
arch/powerpc/kernel/epapr_hcalls.S
View File

@ -21,10 +21,9 @@
#ifndef CONFIG_PPC64
/* epapr_ev_idle() was derived from e500_idle() */
_GLOBAL(epapr_ev_idle)
CURRENT_THREAD_INFO(r3, r1)
PPC_LL r4, TI_LOCAL_FLAGS(r3) /* set napping bit */
PPC_LL r4, TI_LOCAL_FLAGS(r2) /* set napping bit */
ori r4, r4,_TLF_NAPPING /* so when we take an exception */
PPC_STL r4, TI_LOCAL_FLAGS(r3) /* it will return to our caller */
PPC_STL r4, TI_LOCAL_FLAGS(r2) /* it will return to our caller */
wrteei 1

14
arch/powerpc/kernel/exceptions-64e.S
View File

@ -77,17 +77,6 @@ special_reg_save:
andi. r3,r3,MSR_PR
bnelr
/* Copy info into temporary exception thread info */
ld r11,PACAKSAVE(r13)
CURRENT_THREAD_INFO(r11, r11)
CURRENT_THREAD_INFO(r12, r1)
ld r10,TI_FLAGS(r11)
std r10,TI_FLAGS(r12)
ld r10,TI_PREEMPT(r11)
std r10,TI_PREEMPT(r12)
ld r10,TI_TASK(r11)
std r10,TI_TASK(r12)
/*
* Advance to the next TLB exception frame for handler
* types that don't do it automatically.
@ -349,6 +338,7 @@ ret_from_mc_except:
#define GEN_BTB_FLUSH
#define CRIT_BTB_FLUSH
#define DBG_BTB_FLUSH
#define MC_BTB_FLUSH
#define GDBELL_BTB_FLUSH
#endif
@ -504,7 +494,7 @@ exc_##n##_bad_stack: \
* interrupts happen before the wait instruction.
*/
#define CHECK_NAPPING() \
CURRENT_THREAD_INFO(r11, r1); \
ld r11, PACA_THREAD_INFO(r13); \
ld r10,TI_LOCAL_FLAGS(r11); \
andi. r9,r10,_TLF_NAPPING; \
beq+ 1f; \

94
arch/powerpc/kernel/exceptions-64s.S
View File

@ -68,6 +68,14 @@ OPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900)
OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x4000)
OPEN_FIXED_SECTION(virt_vectors, 0x4000, 0x5900)
OPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000)
#ifdef CONFIG_PPC_POWERNV
.globl start_real_trampolines
.globl end_real_trampolines
.globl start_virt_trampolines
.globl end_virt_trampolines
#endif
#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
/*
* Data area reserved for FWNMI option.
@ -566,8 +574,36 @@ EXC_COMMON_BEGIN(mce_return)
RFI_TO_KERNEL
b .
EXC_REAL(data_access, 0x300, 0x80)
EXC_VIRT(data_access, 0x4300, 0x80, 0x300)
EXC_REAL_BEGIN(data_access, 0x300, 0x80)
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_0(PACA_EXGEN)
b tramp_real_data_access
EXC_REAL_END(data_access, 0x300, 0x80)
TRAMP_REAL_BEGIN(tramp_real_data_access)
EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x300)
/*
* DAR/DSISR must be read before setting MSR[RI], because
* a d-side MCE will clobber those registers so is not
* recoverable if they are live.
*/
mfspr r10,SPRN_DAR
mfspr r11,SPRN_DSISR
std r10,PACA_EXGEN+EX_DAR(r13)
stw r11,PACA_EXGEN+EX_DSISR(r13)
EXCEPTION_PROLOG_2(data_access_common, EXC_STD)
EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_0(PACA_EXGEN)
EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x300)
mfspr r10,SPRN_DAR
mfspr r11,SPRN_DSISR
std r10,PACA_EXGEN+EX_DAR(r13)
stw r11,PACA_EXGEN+EX_DSISR(r13)
EXCEPTION_PROLOG_2_RELON(data_access_common, EXC_STD)
EXC_VIRT_END(data_access, 0x4300, 0x80)
TRAMP_KVM_SKIP(PACA_EXGEN, 0x300)
EXC_COMMON_BEGIN(data_access_common)
@ -575,11 +611,8 @@ EXC_COMMON_BEGIN(data_access_common)
* Here r13 points to the paca, r9 contains the saved CR,
* SRR0 and SRR1 are saved in r11 and r12,
* r9 - r13 are saved in paca->exgen.
* EX_DAR and EX_DSISR have saved DAR/DSISR
*/
mfspr r10,SPRN_DAR
std r10,PACA_EXGEN+EX_DAR(r13)
mfspr r10,SPRN_DSISR
stw r10,PACA_EXGEN+EX_DSISR(r13)
EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
RECONCILE_IRQ_STATE(r10, r11)
ld r12,_MSR(r1)
@ -596,18 +629,29 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
EXCEPTION_PROLOG(PACA_EXSLB, data_access_slb_common, EXC_STD, KVMTEST_PR, 0x380);
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_0(PACA_EXSLB)
b tramp_real_data_access_slb
EXC_REAL_END(data_access_slb, 0x380, 0x80)
TRAMP_REAL_BEGIN(tramp_real_data_access_slb)
EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380)
mfspr r10,SPRN_DAR
std r10,PACA_EXSLB+EX_DAR(r13)
EXCEPTION_PROLOG_2(data_access_slb_common, EXC_STD)
EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
EXCEPTION_RELON_PROLOG(PACA_EXSLB, data_access_slb_common, EXC_STD, NOTEST, 0x380);
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_0(PACA_EXSLB)
EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
mfspr r10,SPRN_DAR
std r10,PACA_EXSLB+EX_DAR(r13)
EXCEPTION_PROLOG_2_RELON(data_access_slb_common, EXC_STD)
EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
TRAMP_KVM_SKIP(PACA_EXSLB, 0x380)
EXC_COMMON_BEGIN(data_access_slb_common)
mfspr r10,SPRN_DAR
std r10,PACA_EXSLB+EX_DAR(r13)
EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB)
ld r4,PACA_EXSLB+EX_DAR(r13)
std r4,_DAR(r1)
@ -703,14 +747,30 @@ TRAMP_KVM_HV(PACA_EXGEN, 0x500)
EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ)
EXC_REAL(alignment, 0x600, 0x100)
EXC_VIRT(alignment, 0x4600, 0x100, 0x600)
EXC_REAL_BEGIN(alignment, 0x600, 0x100)
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_0(PACA_EXGEN)
EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x600)
mfspr r10,SPRN_DAR
mfspr r11,SPRN_DSISR
std r10,PACA_EXGEN+EX_DAR(r13)
stw r11,PACA_EXGEN+EX_DSISR(r13)
EXCEPTION_PROLOG_2(alignment_common, EXC_STD)
EXC_REAL_END(alignment, 0x600, 0x100)
EXC_VIRT_BEGIN(alignment, 0x4600, 0x100)
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_0(PACA_EXGEN)
EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x600)
mfspr r10,SPRN_DAR
mfspr r11,SPRN_DSISR
std r10,PACA_EXGEN+EX_DAR(r13)
stw r11,PACA_EXGEN+EX_DSISR(r13)
EXCEPTION_PROLOG_2_RELON(alignment_common, EXC_STD)
EXC_VIRT_END(alignment, 0x4600, 0x100)
TRAMP_KVM(PACA_EXGEN, 0x600)
EXC_COMMON_BEGIN(alignment_common)
mfspr r10,SPRN_DAR
std r10,PACA_EXGEN+EX_DAR(r13)
mfspr r10,SPRN_DSISR
stw r10,PACA_EXGEN+EX_DSISR(r13)
EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
ld r3,PACA_EXGEN+EX_DAR(r13)
lwz r4,PACA_EXGEN+EX_DSISR(r13)
@ -1629,7 +1689,7 @@ do_hash_page:
ori r0,r0,DSISR_BAD_FAULT_64S@l
and. r0,r4,r0 /* weird error? */
bne- handle_page_fault /* if not, try to insert a HPTE */
CURRENT_THREAD_INFO(r11, r1)
ld r11, PACA_THREAD_INFO(r13)
lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */
andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */
bne 77f /* then don't call hash_page now */

160
arch/powerpc/kernel/head_32.S
View File

@ -261,7 +261,7 @@ __secondary_hold_acknowledge:
tophys(r11,r1); /* use tophys(r1) if kernel */ \
beq 1f; \
mfspr r11,SPRN_SPRG_THREAD; \
lwz r11,THREAD_INFO-THREAD(r11); \
lwz r11,TASK_STACK-THREAD(r11); \
addi r11,r11,THREAD_SIZE; \
tophys(r11,r11); \
1: subi r11,r11,INT_FRAME_SIZE /* alloc exc. frame */
@ -352,9 +352,8 @@ i##n: \
* registers that might have bad values includes all the GPRs
* and all the BATs. We indicate that we are in RTAS by putting
* a non-zero value, the address of the exception frame to use,
* in SPRG2. The machine check handler checks SPRG2 and uses its
* value if it is non-zero. If we ever needed to free up SPRG2,
* we could use a field in the thread_info or thread_struct instead.
* in thread.rtas_sp. The machine check handler checks thread.rtas_sp
* and uses its value if it is non-zero.
* (Other exception handlers assume that r1 is a valid kernel stack
* pointer when we take an exception from supervisor mode.)
* -- paulus.
@ -365,16 +364,15 @@ i##n: \
mtspr SPRN_SPRG_SCRATCH1,r11
mfcr r10
#ifdef CONFIG_PPC_CHRP
mfspr r11,SPRN_SPRG_RTAS
cmpwi 0,r11,0
bne 7f
mfspr r11, SPRN_SPRG_THREAD
lwz r11, RTAS_SP(r11)
cmpwi cr1, r11, 0
bne cr1, 7f
#endif /* CONFIG_PPC_CHRP */
EXCEPTION_PROLOG_1
7: EXCEPTION_PROLOG_2
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_PPC_CHRP
mfspr r4,SPRN_SPRG_RTAS
cmpwi cr1,r4,0
bne cr1,1f
#endif
EXC_XFER_STD(0x200, machine_check_exception)
@ -500,18 +498,22 @@ InstructionTLBMiss:
*/
/* Get PTE (linux-style) and check access */
mfspr r3,SPRN_IMISS
#if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC)
lis r1,PAGE_OFFSET@h /* check if kernel address */
cmplw 0,r1,r3
mfspr r2,SPRN_SPRG_THREAD
li r1,_PAGE_USER|_PAGE_PRESENT|_PAGE_EXEC /* low addresses tested as user */
lwz r2,PGDIR(r2)
#endif
mfspr r2, SPRN_SPRG_PGDIR
#ifdef CONFIG_SWAP
li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
#else
li r1,_PAGE_PRESENT | _PAGE_EXEC
#endif
#if defined(CONFIG_MODULES) || defined(CONFIG_DEBUG_PAGEALLOC)
bge- 112f
mfspr r2,SPRN_SRR1 /* and MSR_PR bit from SRR1 */
rlwimi r1,r2,32-12,29,29 /* shift MSR_PR to _PAGE_USER posn */
lis r2,swapper_pg_dir@ha /* if kernel address, use */
addi r2,r2,swapper_pg_dir@l /* kernel page table */
112: tophys(r2,r2)
rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
#endif
112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
lwz r2,0(r2) /* get pmd entry */
rlwinm. r2,r2,0,0,19 /* extract address of pte page */
beq- InstructionAddressInvalid /* return if no mapping */
@ -519,20 +521,10 @@ InstructionTLBMiss:
lwz r0,0(r2) /* get linux-style pte */
andc. r1,r1,r0 /* check access & ~permission */
bne- InstructionAddressInvalid /* return if access not permitted */
ori r0,r0,_PAGE_ACCESSED /* set _PAGE_ACCESSED in pte */
/*
* NOTE! We are assuming this is not an SMP system, otherwise
* we would need to update the pte atomically with lwarx/stwcx.
*/
stw r0,0(r2) /* update PTE (accessed bit) */
/* Convert linux-style PTE to low word of PPC-style PTE */
rlwinm r1,r0,32-10,31,31 /* _PAGE_RW -> PP lsb */
rlwinm r2,r0,32-7,31,31 /* _PAGE_DIRTY -> PP lsb */
and r1,r1,r2 /* writable if _RW and _DIRTY */
rlwimi r0,r0,32-1,30,30 /* _PAGE_USER -> PP msb */
rlwimi r0,r0,32-1,31,31 /* _PAGE_USER -> PP lsb */
ori r1,r1,0xe04 /* clear out reserved bits */
andc r1,r0,r1 /* PP = user? (rw&dirty? 2: 3): 0 */
ori r1, r1, 0xe05 /* clear out reserved bits */
andc r1, r0, r1 /* PP = user? 2 : 0 */
BEGIN_FTR_SECTION
rlwinm r1,r1,0,~_PAGE_COHERENT /* clear M (coherence not required) */
END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
@ -576,16 +568,16 @@ DataLoadTLBMiss:
mfspr r3,SPRN_DMISS
lis r1,PAGE_OFFSET@h /* check if kernel address */
cmplw 0,r1,r3
mfspr r2,SPRN_SPRG_THREAD
li r1,_PAGE_USER|_PAGE_PRESENT /* low addresses tested as user */
lwz r2,PGDIR(r2)
mfspr r2, SPRN_SPRG_PGDIR
#ifdef CONFIG_SWAP
li r1, _PAGE_PRESENT | _PAGE_ACCESSED
#else
li r1, _PAGE_PRESENT
#endif
bge- 112f
mfspr r2,SPRN_SRR1 /* and MSR_PR bit from SRR1 */
rlwimi r1,r2,32-12,29,29 /* shift MSR_PR to _PAGE_USER posn */
lis r2,swapper_pg_dir@ha /* if kernel address, use */
addi r2,r2,swapper_pg_dir@l /* kernel page table */
112: tophys(r2,r2)
rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
lwz r2,0(r2) /* get pmd entry */
rlwinm. r2,r2,0,0,19 /* extract address of pte page */
beq- DataAddressInvalid /* return if no mapping */
@ -593,20 +585,16 @@ DataLoadTLBMiss:
lwz r0,0(r2) /* get linux-style pte */
andc. r1,r1,r0 /* check access & ~permission */
bne- DataAddressInvalid /* return if access not permitted */
ori r0,r0,_PAGE_ACCESSED /* set _PAGE_ACCESSED in pte */
/*
* NOTE! We are assuming this is not an SMP system, otherwise
* we would need to update the pte atomically with lwarx/stwcx.
*/
stw r0,0(r2) /* update PTE (accessed bit) */
/* Convert linux-style PTE to low word of PPC-style PTE */
rlwinm r1,r0,32-10,31,31 /* _PAGE_RW -> PP lsb */
rlwinm r2,r0,32-7,31,31 /* _PAGE_DIRTY -> PP lsb */
and r1,r1,r2 /* writable if _RW and _DIRTY */
rlwimi r0,r0,32-1,30,30 /* _PAGE_USER -> PP msb */
rlwimi r0,r0,32-1,31,31 /* _PAGE_USER -> PP lsb */
ori r1,r1,0xe04 /* clear out reserved bits */
andc r1,r0,r1 /* PP = user? (rw&dirty? 2: 3): 0 */
andc r1,r0,r1 /* PP = user? rw? 2: 3: 0 */
BEGIN_FTR_SECTION
rlwinm r1,r1,0,~_PAGE_COHERENT /* clear M (coherence not required) */
END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
@ -660,16 +648,16 @@ DataStoreTLBMiss:
mfspr r3,SPRN_DMISS
lis r1,PAGE_OFFSET@h /* check if kernel address */
cmplw 0,r1,r3
mfspr r2,SPRN_SPRG_THREAD
li r1,_PAGE_RW|_PAGE_USER|_PAGE_PRESENT /* access flags */
lwz r2,PGDIR(r2)
mfspr r2, SPRN_SPRG_PGDIR
#ifdef CONFIG_SWAP
li r1, _PAGE_RW | _PAGE_PRESENT | _PAGE_ACCESSED
#else
li r1, _PAGE_RW | _PAGE_PRESENT
#endif
bge- 112f
mfspr r2,SPRN_SRR1 /* and MSR_PR bit from SRR1 */
rlwimi r1,r2,32-12,29,29 /* shift MSR_PR to _PAGE_USER posn */
lis r2,swapper_pg_dir@ha /* if kernel address, use */
addi r2,r2,swapper_pg_dir@l /* kernel page table */
112: tophys(r2,r2)
rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */
lwz r2,0(r2) /* get pmd entry */
rlwinm. r2,r2,0,0,19 /* extract address of pte page */
beq- DataAddressInvalid /* return if no mapping */
@ -677,12 +665,10 @@ DataStoreTLBMiss:
lwz r0,0(r2) /* get linux-style pte */
andc. r1,r1,r0 /* check access & ~permission */
bne- DataAddressInvalid /* return if access not permitted */
ori r0,r0,_PAGE_ACCESSED|_PAGE_DIRTY
/*
* NOTE! We are assuming this is not an SMP system, otherwise
* we would need to update the pte atomically with lwarx/stwcx.
*/
stw r0,0(r2) /* update PTE (accessed/dirty bits) */
/* Convert linux-style PTE to low word of PPC-style PTE */
rlwimi r0,r0,32-1,30,30 /* _PAGE_USER -> PP msb */
li r1,0xe05 /* clear out reserved bits & PP lsb */
@ -845,12 +831,12 @@ __secondary_start:
bl init_idle_6xx
#endif /* CONFIG_PPC_BOOK3S_32 */
/* get current_thread_info and current */
lis r1,secondary_ti@ha
tophys(r1,r1)
lwz r1,secondary_ti@l(r1)
tophys(r2,r1)
lwz r2,TI_TASK(r2)
/* get current's stack and current */
lis r2,secondary_current@ha
tophys(r2,r2)
lwz r2,secondary_current@l(r2)
tophys(r1,r2)
lwz r1,TASK_STACK(r1)
/* stack */
addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
@ -865,8 +851,10 @@ __secondary_start:
tophys(r4,r2)
addi r4,r4,THREAD /* phys address of our thread_struct */
mtspr SPRN_SPRG_THREAD,r4
#ifdef CONFIG_PPC_RTAS
li r3,0
mtspr SPRN_SPRG_RTAS,r3 /* 0 => not in RTAS */
stw r3, RTAS_SP(r4) /* 0 => not in RTAS */
#endif
/* enable MMU and jump to start_secondary */
li r4,MSR_KERNEL
@ -950,8 +938,10 @@ start_here:
tophys(r4,r2)
addi r4,r4,THREAD /* init task's THREAD */
mtspr SPRN_SPRG_THREAD,r4
#ifdef CONFIG_PPC_RTAS
li r3,0
mtspr SPRN_SPRG_RTAS,r3 /* 0 => not in RTAS */
stw r3, RTAS_SP(r4) /* 0 => not in RTAS */
#endif
/* stack */
lis r1,init_thread_union@ha
@ -1022,15 +1012,16 @@ _ENTRY(switch_mmu_context)
li r0,NUM_USER_SEGMENTS
mtctr r0
lwz r4, MM_PGD(r4)
#ifdef CONFIG_BDI_SWITCH
/* Context switch the PTE pointer for the Abatron BDI2000.
* The PGDIR is passed as second argument.
*/
lwz r4,MM_PGD(r4)
lis r5, KERNELBASE@h
lwz r5, 0xf0(r5)
stw r4, 0x4(r5)
lis r5, abatron_pteptrs@ha
stw r4, abatron_pteptrs@l + 0x4(r5)
#endif
tophys(r4, r4)
mtspr SPRN_SPRG_PGDIR, r4
li r4,0
isync
3:
@ -1105,6 +1096,41 @@ BEGIN_MMU_FTR_SECTION
END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
blr
_ENTRY(update_bats)
lis r4, 1f@h
ori r4, r4, 1f@l
tophys(r4, r4)
mfmsr r6
mflr r7
li r3, MSR_KERNEL & ~(MSR_IR | MSR_DR)
rlwinm r0, r6, 0, ~MSR_RI
rlwinm r0, r0, 0, ~MSR_EE
mtmsr r0
mtspr SPRN_SRR0, r4
mtspr SPRN_SRR1, r3
SYNC
RFI
1: bl clear_bats
lis r3, BATS@ha
addi r3, r3, BATS@l
tophys(r3, r3)
LOAD_BAT(0, r3, r4, r5)
LOAD_BAT(1, r3, r4, r5)
LOAD_BAT(2, r3, r4, r5)
LOAD_BAT(3, r3, r4, r5)
BEGIN_MMU_FTR_SECTION
LOAD_BAT(4, r3, r4, r5)
LOAD_BAT(5, r3, r4, r5)
LOAD_BAT(6, r3, r4, r5)
LOAD_BAT(7, r3, r4, r5)
END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
li r3, MSR_KERNEL & ~(MSR_IR | MSR_DR | MSR_RI)
mtmsr r3
mtspr SPRN_SRR0, r7
mtspr SPRN_SRR1, r6
SYNC
RFI
flush_tlbs:
lis r10, 0x40
1: addic. r10, r10, -0x1000

9
arch/powerpc/kernel/head_40x.S
View File

@ -115,7 +115,7 @@ _ENTRY(saved_ksp_limit)
andi. r11,r11,MSR_PR; \
beq 1f; \
mfspr r1,SPRN_SPRG_THREAD; /* if from user, start at top of */\
lwz r1,THREAD_INFO-THREAD(r1); /* this thread's kernel stack */\
lwz r1,TASK_STACK-THREAD(r1); /* this thread's kernel stack */\
addi r1,r1,THREAD_SIZE; \
1: subi r1,r1,INT_FRAME_SIZE; /* Allocate an exception frame */\
tophys(r11,r1); \
@ -158,7 +158,7 @@ _ENTRY(saved_ksp_limit)
beq 1f; \
/* COMING FROM USER MODE */ \
mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\
lwz r11,THREAD_INFO-THREAD(r11); /* this thread's kernel stack */\
lwz r11,TASK_STACK-THREAD(r11); /* this thread's kernel stack */\
1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE; /* Alloc an excpt frm */\
tophys(r11,r11); \
stw r10,_CCR(r11); /* save various registers */\
@ -953,9 +953,8 @@ _GLOBAL(set_context)
/* Context switch the PTE pointer for the Abatron BDI2000.
* The PGDIR is the second parameter.
*/
lis r5, KERNELBASE@h
lwz r5, 0xf0(r5)
stw r4, 0x4(r5)
lis r5, abatron_pteptrs@ha
stw r4, abatron_pteptrs@l + 0x4(r5)
#endif
sync
mtspr SPRN_PID,r3

8
arch/powerpc/kernel/head_44x.S
View File

@ -1019,10 +1019,10 @@ _GLOBAL(start_secondary_47x)
/* Now we can get our task struct and real stack pointer */
/* Get current_thread_info and current */
lis r1,secondary_ti@ha
lwz r1,secondary_ti@l(r1)
lwz r2,TI_TASK(r1)
/* Get current's stack and current */
lis r2,secondary_current@ha
lwz r2,secondary_current@l(r2)
lwz r1,TASK_STACK(r2)
/* Current stack pointer */
addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD

20
arch/powerpc/kernel/head_64.S
View File

@ -801,21 +801,19 @@ __secondary_start:
/* Set thread priority to MEDIUM */
HMT_MEDIUM
/* Initialize the kernel stack */
LOAD_REG_ADDR(r3, current_set)
sldi r28,r24,3 /* get current_set[cpu#] */
ldx r14,r3,r28
addi r14,r14,THREAD_SIZE-STACK_FRAME_OVERHEAD
std r14,PACAKSAVE(r13)
/* Do early setup for that CPU (SLB and hash table pointer) */
/*
* Do early setup for this CPU, in particular initialising the MMU so we
* can turn it on below. This is a call to C, which is OK, we're still
* running on the emergency stack.
*/
bl early_setup_secondary
/*
* setup the new stack pointer, but *don't* use this until
* translation is on.
* The primary has initialized our kernel stack for us in the paca, grab
* it and put it in r1. We must *not* use it until we turn on the MMU
* below, because it may not be inside the RMO.
*/
mr r1, r14
ld r1, PACAKSAVE(r13)
/* Clear backchain so we get nice backtraces */
li r7,0

124
arch/powerpc/kernel/head_8xx.S
View File

@ -142,7 +142,7 @@ instruction_counter:
tophys(r11,r1); /* use tophys(r1) if kernel */ \
beq 1f; \
mfspr r11,SPRN_SPRG_THREAD; \
lwz r11,THREAD_INFO-THREAD(r11); \
lwz r11,TASK_STACK-THREAD(r11); \
addi r11,r11,THREAD_SIZE; \
tophys(r11,r11); \
1: subi r11,r11,INT_FRAME_SIZE /* alloc exc. frame */
@ -292,6 +292,17 @@ SystemCall:
*/
EXCEPTION(0x1000, SoftEmu, program_check_exception, EXC_XFER_STD)
/* Called from DataStoreTLBMiss when perf TLB misses events are activated */
#ifdef CONFIG_PERF_EVENTS
patch_site 0f, patch__dtlbmiss_perf
0: lwz r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
addi r10, r10, 1
stw r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
mfspr r10, SPRN_SPRG_SCRATCH0
mfspr r11, SPRN_SPRG_SCRATCH1
rfi
#endif
. = 0x1100
/*
* For the MPC8xx, this is a software tablewalk to load the instruction
@ -337,8 +348,8 @@ InstructionTLBMiss:
rlwinm r10, r10, 16, 0xfff8
cmpli cr0, r10, PAGE_OFFSET@h
#ifndef CONFIG_PIN_TLB_TEXT
/* It is assumed that kernel code fits into the first 8M page */
0: cmpli cr7, r10, (PAGE_OFFSET + 0x0800000)@h
/* It is assumed that kernel code fits into the first 32M */
0: cmpli cr7, r10, (PAGE_OFFSET + 0x2000000)@h
patch_site 0b, patch__itlbmiss_linmem_top
#endif
#endif
@ -405,10 +416,20 @@ InstructionTLBMiss:
#ifndef CONFIG_PIN_TLB_TEXT
ITLBMissLinear:
mtcr r11
#if defined(CONFIG_STRICT_KERNEL_RWX) && CONFIG_ETEXT_SHIFT < 23
patch_site 0f, patch__itlbmiss_linmem_top8
mfspr r10, SPRN_SRR0
0: subis r11, r10, (PAGE_OFFSET - 0x80000000)@ha
rlwinm r11, r11, 4, MI_PS8MEG ^ MI_PS512K
ori r11, r11, MI_PS512K | MI_SVALID
rlwinm r10, r10, 0, 0x0ff80000 /* 8xx supports max 256Mb RAM */
#else
/* Set 8M byte page and mark it valid */
li r11, MI_PS8MEG | MI_SVALID
mtspr SPRN_MI_TWC, r11
rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
#endif
mtspr SPRN_MI_TWC, r11
ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
_PAGE_PRESENT
mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
@ -434,7 +455,7 @@ DataStoreTLBMiss:
#ifndef CONFIG_PIN_TLB_IMMR
cmpli cr6, r10, VIRT_IMMR_BASE@h
#endif
0: cmpli cr7, r10, (PAGE_OFFSET + 0x1800000)@h
0: cmpli cr7, r10, (PAGE_OFFSET + 0x2000000)@h
patch_site 0b, patch__dtlbmiss_linmem_top
mfspr r10, SPRN_M_TWB /* Get level 1 table */
@ -494,16 +515,6 @@ DataStoreTLBMiss:
rfi
patch_site 0b, patch__dtlbmiss_exit_1
#ifdef CONFIG_PERF_EVENTS
patch_site 0f, patch__dtlbmiss_perf
0: lwz r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
addi r10, r10, 1
stw r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0)
mfspr r10, SPRN_SPRG_SCRATCH0
mfspr r11, SPRN_SPRG_SCRATCH1
rfi
#endif
DTLBMissIMMR:
mtcr r11
/* Set 512k byte guarded page and mark it valid */
@ -525,10 +536,29 @@ DTLBMissIMMR:
DTLBMissLinear:
mtcr r11
rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
#if defined(CONFIG_STRICT_KERNEL_RWX) && CONFIG_DATA_SHIFT < 23
patch_site 0f, patch__dtlbmiss_romem_top8
0: subis r11, r10, (PAGE_OFFSET - 0x80000000)@ha
rlwinm r11, r11, 0, 0xff800000
neg r10, r11
or r11, r11, r10
rlwinm r11, r11, 4, MI_PS8MEG ^ MI_PS512K
ori r11, r11, MI_PS512K | MI_SVALID
mfspr r10, SPRN_MD_EPN
rlwinm r10, r10, 0, 0x0ff80000 /* 8xx supports max 256Mb RAM */
#else
/* Set 8M byte page and mark it valid */
li r11, MD_PS8MEG | MD_SVALID
#endif
mtspr SPRN_MD_TWC, r11
rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
#ifdef CONFIG_STRICT_KERNEL_RWX
patch_site 0f, patch__dtlbmiss_romem_top
0: subis r11, r10, 0
rlwimi r10, r11, 11, _PAGE_RO
#endif
ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
_PAGE_PRESENT
mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
@ -551,11 +581,11 @@ InstructionTLBError:
mr r4,r12
andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
andis. r10,r9,SRR1_ISI_NOPT@h
beq+ 1f
beq+ .Litlbie
tlbie r4
itlbie:
/* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
1: EXC_XFER_LITE(0x400, handle_page_fault)
.Litlbie:
EXC_XFER_LITE(0x400, handle_page_fault)
/* This is the data TLB error on the MPC8xx. This could be due to
* many reasons, including a dirty update to a pte. We bail out to
@ -577,10 +607,10 @@ DARFixed:/* Return from dcbx instruction bug workaround */
stw r5,_DSISR(r11)
mfspr r4,SPRN_DAR
andis. r10,r5,DSISR_NOHPTE@h
beq+ 1f
beq+ .Ldtlbie
tlbie r4
dtlbie:
1: li r10,RPN_PATTERN
.Ldtlbie:
li r10,RPN_PATTERN
mtspr SPRN_DAR,r10 /* Tag DAR, to be used in DTLB Error */
/* 0x300 is DataAccess exception, needed by bad_page_fault() */
EXC_XFER_LITE(0x300, handle_page_fault)
@ -603,8 +633,8 @@ DataBreakpoint:
mtspr SPRN_SPRG_SCRATCH1, r11
mfcr r10
mfspr r11, SPRN_SRR0
cmplwi cr0, r11, (dtlbie - PAGE_OFFSET)@l
cmplwi cr7, r11, (itlbie - PAGE_OFFSET)@l
cmplwi cr0, r11, (.Ldtlbie - PAGE_OFFSET)@l
cmplwi cr7, r11, (.Litlbie - PAGE_OFFSET)@l
beq- cr0, 11f
beq- cr7, 11f
EXCEPTION_PROLOG_1
@ -886,28 +916,11 @@ initial_mmu:
mtspr SPRN_MD_CTR, r10 /* remove PINNED DTLB entries */
tlbia /* Invalidate all TLB entries */
#ifdef CONFIG_PIN_TLB_TEXT
lis r8, MI_RSV4I@h
ori r8, r8, 0x1c00
mtspr SPRN_MI_CTR, r8 /* Set instruction MMU control */
#endif
#ifdef CONFIG_PIN_TLB_DATA
oris r10, r10, MD_RSV4I@h
mtspr SPRN_MD_CTR, r10 /* Set data TLB control */
#endif
/* Now map the lower 8 Meg into the ITLB. */
lis r8, KERNELBASE@h /* Create vaddr for TLB */
ori r8, r8, MI_EVALID /* Mark it valid */
mtspr SPRN_MI_EPN, r8
li r8, MI_PS8MEG /* Set 8M byte page */
ori r8, r8, MI_SVALID /* Make it valid */
mtspr SPRN_MI_TWC, r8
li r8, MI_BOOTINIT /* Create RPN for address 0 */
mtspr SPRN_MI_RPN, r8 /* Store TLB entry */
lis r8, MI_APG_INIT@h /* Set protection modes */
ori r8, r8, MI_APG_INIT@l
mtspr SPRN_MI_AP, r8
@ -937,6 +950,34 @@ initial_mmu:
mtspr SPRN_MD_RPN, r8
#endif
/* Now map the lower RAM (up to 32 Mbytes) into the ITLB. */
#ifdef CONFIG_PIN_TLB_TEXT
lis r8, MI_RSV4I@h
ori r8, r8, 0x1c00
#endif
li r9, 4 /* up to 4 pages of 8M */
mtctr r9
lis r9, KERNELBASE@h /* Create vaddr for TLB */
li r10, MI_PS8MEG | MI_SVALID /* Set 8M byte page */
li r11, MI_BOOTINIT /* Create RPN for address 0 */
lis r12, _einittext@h
ori r12, r12, _einittext@l
1:
#ifdef CONFIG_PIN_TLB_TEXT
mtspr SPRN_MI_CTR, r8 /* Set instruction MMU control */
addi r8, r8, 0x100
#endif
ori r0, r9, MI_EVALID /* Mark it valid */
mtspr SPRN_MI_EPN, r0
mtspr SPRN_MI_TWC, r10
mtspr SPRN_MI_RPN, r11 /* Store TLB entry */
addis r9, r9, 0x80
addis r11, r11, 0x80
cmpl cr0, r9, r12
bdnzf gt, 1b
/* Since the cache is enabled according to the information we
* just loaded into the TLB, invalidate and enable the caches here.
* We should probably check/set other modes....later.
@ -989,5 +1030,6 @@ swapper_pg_dir:
/* Room for two PTE table poiners, usually the kernel and current user
* pointer to their respective root page table (pgdir).
*/
.globl abatron_pteptrs
abatron_pteptrs:
.space 8

12
arch/powerpc/kernel/head_booke.h
View File

@ -55,7 +55,7 @@ END_BTB_FLUSH_SECTION
beq 1f; \
BOOKE_CLEAR_BTB(r11) \
/* if from user, start at top of this thread's kernel stack */ \
lwz r11, THREAD_INFO-THREAD(r10); \
lwz r11, TASK_STACK - THREAD(r10); \
ALLOC_STACK_FRAME(r11, THREAD_SIZE); \
1 : subi r11, r11, INT_FRAME_SIZE; /* Allocate exception frame */ \
stw r13, _CCR(r11); /* save various registers */ \
@ -142,7 +142,7 @@ END_BTB_FLUSH_SECTION
BOOKE_CLEAR_BTB(r10) \
andi. r11,r11,MSR_PR; \
mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\
lwz r11,THREAD_INFO-THREAD(r11); /* this thread's kernel stack */\
lwz r11, TASK_STACK - THREAD(r11); /* this thread's kernel stack */\
addi r11,r11,EXC_LVL_FRAME_OVERHEAD; /* allocate stack frame */\
beq 1f; \
/* COMING FROM USER MODE */ \
@ -155,13 +155,7 @@ END_BTB_FLUSH_SECTION
stw r10,GPR11(r11); \
b 2f; \
/* COMING FROM PRIV MODE */ \
1: lwz r9,TI_FLAGS-EXC_LVL_FRAME_OVERHEAD(r11); \
lwz r10,TI_PREEMPT-EXC_LVL_FRAME_OVERHEAD(r11); \
stw r9,TI_FLAGS-EXC_LVL_FRAME_OVERHEAD(r8); \
stw r10,TI_PREEMPT-EXC_LVL_FRAME_OVERHEAD(r8); \
lwz r9,TI_TASK-EXC_LVL_FRAME_OVERHEAD(r11); \
stw r9,TI_TASK-EXC_LVL_FRAME_OVERHEAD(r8); \
mr r11,r8; \
1: mr r11, r8; \
2: mfspr r8,SPRN_SPRG_RSCRATCH_##exc_level; \
stw r12,GPR12(r11); /* save various registers */\
mflr r10; \

16
arch/powerpc/kernel/head_fsl_booke.S
View File

@ -243,8 +243,9 @@ set_ivor:
li r0,0
stwu r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
CURRENT_THREAD_INFO(r22, r1)
stw r24, TI_CPU(r22)
#ifdef CONFIG_SMP
stw r24, TASK_CPU(r2)
#endif
bl early_init
@ -717,8 +718,7 @@ finish_tlb_load:
/* Get the next_tlbcam_idx percpu var */
#ifdef CONFIG_SMP
lwz r12, THREAD_INFO-THREAD(r12)
lwz r15, TI_CPU(r12)
lwz r15, TASK_CPU-THREAD(r12)
lis r14, __per_cpu_offset@h
ori r14, r14, __per_cpu_offset@l
rlwinm r15, r15, 2, 0, 29
@ -1089,10 +1089,10 @@ __secondary_start:
mr r4,r24 /* Why? */
bl call_setup_cpu
/* get current_thread_info and current */
lis r1,secondary_ti@ha
lwz r1,secondary_ti@l(r1)
lwz r2,TI_TASK(r1)
/* get current's stack and current */
lis r2,secondary_current@ha
lwz r2,secondary_current@l(r2)
lwz r1,TASK_STACK(r2)
/* stack */
addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD

8
arch/powerpc/kernel/idle_6xx.S
View File

@ -136,10 +136,9 @@ BEGIN_FTR_SECTION
DSSALL
sync
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
CURRENT_THREAD_INFO(r9, r1)
lwz r8,TI_LOCAL_FLAGS(r9) /* set napping bit */
lwz r8,TI_LOCAL_FLAGS(r2) /* set napping bit */
ori r8,r8,_TLF_NAPPING /* so when we take an exception */
stw r8,TI_LOCAL_FLAGS(r9) /* it will return to our caller */
stw r8,TI_LOCAL_FLAGS(r2) /* it will return to our caller */
mfmsr r7
ori r7,r7,MSR_EE
oris r7,r7,MSR_POW@h
@ -159,8 +158,7 @@ _GLOBAL(power_save_ppc32_restore)
stw r9,_NIP(r11) /* make it do a blr */
#ifdef CONFIG_SMP
CURRENT_THREAD_INFO(r12, r11)
lwz r11,TI_CPU(r12) /* get cpu number * 4 */
lwz r11,TASK_CPU(r2) /* get cpu number * 4 */
slwi r11,r11,2
#else
li r11,0

2
arch/powerpc/kernel/idle_book3e.S
View File

@ -63,7 +63,7 @@ _GLOBAL(\name)
1: /* Let's set the _TLF_NAPPING flag so interrupts make us return
* to the right spot
*/
CURRENT_THREAD_INFO(r11, r1)
ld r11, PACACURRENT(r13)
ld r10,TI_LOCAL_FLAGS(r11)
ori r10,r10,_TLF_NAPPING
std r10,TI_LOCAL_FLAGS(r11)

8
arch/powerpc/kernel/idle_e500.S
View File

@ -22,10 +22,9 @@
.text
_GLOBAL(e500_idle)
CURRENT_THREAD_INFO(r3, r1)
lwz r4,TI_LOCAL_FLAGS(r3) /* set napping bit */
lwz r4,TI_LOCAL_FLAGS(r2) /* set napping bit */
ori r4,r4,_TLF_NAPPING /* so when we take an exception */
stw r4,TI_LOCAL_FLAGS(r3) /* it will return to our caller */
stw r4,TI_LOCAL_FLAGS(r2) /* it will return to our caller */
#ifdef CONFIG_PPC_E500MC
wrteei 1
@ -88,8 +87,7 @@ _GLOBAL(power_save_ppc32_restore)
stw r9,_NIP(r11) /* make it do a blr */
#ifdef CONFIG_SMP
CURRENT_THREAD_INFO(r12, r1)
lwz r11,TI_CPU(r12) /* get cpu number * 4 */
lwz r11,TASK_CPU(r2) /* get cpu number * 4 */
slwi r11,r11,2
#else
li r11,0

2
arch/powerpc/kernel/idle_power4.S
View File

@ -68,7 +68,7 @@ BEGIN_FTR_SECTION
DSSALL
sync
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
CURRENT_THREAD_INFO(r9, r1)
ld r9, PACA_THREAD_INFO(r13)
ld r8,TI_LOCAL_FLAGS(r9) /* set napping bit */
ori r8,r8,_TLF_NAPPING /* so when we take an exception */
std r8,TI_LOCAL_FLAGS(r9) /* it will return to our caller */

119
arch/powerpc/kernel/irq.c
View File

@ -618,9 +618,8 @@ static inline void check_stack_overflow(void)
sp = current_stack_pointer() & (THREAD_SIZE-1);
/* check for stack overflow: is there less than 2KB free? */
if (unlikely(sp < (sizeof(struct thread_info) + 2048))) {
pr_err("do_IRQ: stack overflow: %ld\n",
sp - sizeof(struct thread_info));
if (unlikely(sp < 2048)) {
pr_err("do_IRQ: stack overflow: %ld\n", sp);
dump_stack();
}
#endif
@ -660,36 +659,21 @@ void __do_irq(struct pt_regs *regs)
void do_IRQ(struct pt_regs *regs)
{
struct pt_regs *old_regs = set_irq_regs(regs);
struct thread_info *curtp, *irqtp, *sirqtp;
void *cursp, *irqsp, *sirqsp;
/* Switch to the irq stack to handle this */
curtp = current_thread_info();
irqtp = hardirq_ctx[raw_smp_processor_id()];
sirqtp = softirq_ctx[raw_smp_processor_id()];
cursp = (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));
irqsp = hardirq_ctx[raw_smp_processor_id()];
sirqsp = softirq_ctx[raw_smp_processor_id()];
/* Already there ? */
if (unlikely(curtp == irqtp || curtp == sirqtp)) {
if (unlikely(cursp == irqsp || cursp == sirqsp)) {
__do_irq(regs);
set_irq_regs(old_regs);
return;
}
/* Prepare the thread_info in the irq stack */
irqtp->task = curtp->task;
irqtp->flags = 0;
/* Copy the preempt_count so that the [soft]irq checks work. */
irqtp->preempt_count = curtp->preempt_count;
/* Switch stack and call */
call_do_irq(regs, irqtp);
/* Restore stack limit */
irqtp->task = NULL;
/* Copy back updates to the thread_info */
if (irqtp->flags)
set_bits(irqtp->flags, &curtp->flags);
call_do_irq(regs, irqsp);
set_irq_regs(old_regs);
}
@ -698,90 +682,20 @@ void __init init_IRQ(void)
{
if (ppc_md.init_IRQ)
ppc_md.init_IRQ();
exc_lvl_ctx_init();
irq_ctx_init();
}
#if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
struct thread_info *critirq_ctx[NR_CPUS] __read_mostly;
struct thread_info *dbgirq_ctx[NR_CPUS] __read_mostly;
struct thread_info *mcheckirq_ctx[NR_CPUS] __read_mostly;
void exc_lvl_ctx_init(void)
{
struct thread_info *tp;
int i, cpu_nr;
for_each_possible_cpu(i) {
#ifdef CONFIG_PPC64
cpu_nr = i;
#else
#ifdef CONFIG_SMP
cpu_nr = get_hard_smp_processor_id(i);
#else
cpu_nr = 0;
#endif
void *critirq_ctx[NR_CPUS] __read_mostly;
void *dbgirq_ctx[NR_CPUS] __read_mostly;
void *mcheckirq_ctx[NR_CPUS] __read_mostly;
#endif
memset((void *)critirq_ctx[cpu_nr], 0, THREAD_SIZE);
tp = critirq_ctx[cpu_nr];
tp->cpu = cpu_nr;
tp->preempt_count = 0;
#ifdef CONFIG_BOOKE
memset((void *)dbgirq_ctx[cpu_nr], 0, THREAD_SIZE);
tp = dbgirq_ctx[cpu_nr];
tp->cpu = cpu_nr;
tp->preempt_count = 0;
memset((void *)mcheckirq_ctx[cpu_nr], 0, THREAD_SIZE);
tp = mcheckirq_ctx[cpu_nr];
tp->cpu = cpu_nr;
tp->preempt_count = HARDIRQ_OFFSET;
#endif
}
}
#endif
struct thread_info *softirq_ctx[NR_CPUS] __read_mostly;
struct thread_info *hardirq_ctx[NR_CPUS] __read_mostly;
void irq_ctx_init(void)
{
struct thread_info *tp;
int i;
for_each_possible_cpu(i) {
memset((void *)softirq_ctx[i], 0, THREAD_SIZE);
tp = softirq_ctx[i];
tp->cpu = i;
klp_init_thread_info(tp);
memset((void *)hardirq_ctx[i], 0, THREAD_SIZE);
tp = hardirq_ctx[i];
tp->cpu = i;
klp_init_thread_info(tp);
}
}
void *softirq_ctx[NR_CPUS] __read_mostly;
void *hardirq_ctx[NR_CPUS] __read_mostly;
void do_softirq_own_stack(void)
{
struct thread_info *curtp, *irqtp;
curtp = current_thread_info();
irqtp = softirq_ctx[smp_processor_id()];
irqtp->task = curtp->task;
irqtp->flags = 0;
call_do_softirq(irqtp);
irqtp->task = NULL;
/* Set any flag that may have been set on the
* alternate stack
*/
if (irqtp->flags)
set_bits(irqtp->flags, &curtp->flags);
call_do_softirq(softirq_ctx[smp_processor_id()]);
}
irq_hw_number_t virq_to_hw(unsigned int virq)
@ -827,11 +741,6 @@ int irq_choose_cpu(const struct cpumask *mask)
}
#endif
int arch_early_irq_init(void)
{
return 0;
}
#ifdef CONFIG_PPC64
static int __init setup_noirqdistrib(char *str)
{

28
arch/powerpc/kernel/kgdb.c
View File

@ -151,41 +151,13 @@ static int kgdb_handle_breakpoint(struct pt_regs *regs)
return 1;
}
static DEFINE_PER_CPU(struct thread_info, kgdb_thread_info);
static int kgdb_singlestep(struct pt_regs *regs)
{
struct thread_info *thread_info, *exception_thread_info;
struct thread_info *backup_current_thread_info =
this_cpu_ptr(&kgdb_thread_info);
if (user_mode(regs))
return 0;
/*
* On Book E and perhaps other processors, singlestep is handled on
* the critical exception stack. This causes current_thread_info()
* to fail, since it it locates the thread_info by masking off
* the low bits of the current stack pointer. We work around
* this issue by copying the thread_info from the kernel stack
* before calling kgdb_handle_exception, and copying it back
* afterwards. On most processors the copy is avoided since
* exception_thread_info == thread_info.
*/
thread_info = (struct thread_info *)(regs->gpr[1] & ~(THREAD_SIZE-1));
exception_thread_info = current_thread_info();
if (thread_info != exception_thread_info) {
/* Save the original current_thread_info. */
memcpy(backup_current_thread_info, exception_thread_info, sizeof *thread_info);
memcpy(exception_thread_info, thread_info, sizeof *thread_info);
}
kgdb_handle_exception(0, SIGTRAP, 0, regs);
if (thread_info != exception_thread_info)
/* Restore current_thread_info lastly. */
memcpy(exception_thread_info, backup_current_thread_info, sizeof *thread_info);
return 1;
}

6
arch/powerpc/kernel/machine_kexec_64.c
View File

@ -317,10 +317,8 @@ void default_machine_kexec(struct kimage *image)
* We setup preempt_count to avoid using VMX in memcpy.
* XXX: the task struct will likely be invalid once we do the copy!
*/
kexec_stack.thread_info.task = current_thread_info()->task;
kexec_stack.thread_info.flags = 0;
kexec_stack.thread_info.preempt_count = HARDIRQ_OFFSET;
kexec_stack.thread_info.cpu = current_thread_info()->cpu;
current_thread_info()->flags = 0;
current_thread_info()->preempt_count = HARDIRQ_OFFSET;
/* We need a static PACA, too; copy this CPU's PACA over and switch to
* it. Also poison per_cpu_offset and NULL lppaca to catch anyone using

11
arch/powerpc/kernel/mce.c
View File

@ -31,6 +31,7 @@
#include <asm/machdep.h>
#include <asm/mce.h>
#include <asm/nmi.h>
static DEFINE_PER_CPU(int, mce_nest_count);
static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event);
@ -301,13 +302,13 @@ static void machine_check_process_queued_event(struct irq_work *work)
while (__this_cpu_read(mce_queue_count) > 0) {
index = __this_cpu_read(mce_queue_count) - 1;
evt = this_cpu_ptr(&mce_event_queue[index]);
machine_check_print_event_info(evt, false);
machine_check_print_event_info(evt, false, false);
__this_cpu_dec(mce_queue_count);
}
}
void machine_check_print_event_info(struct machine_check_event *evt,
bool user_mode)
bool user_mode, bool in_guest)
{
const char *level, *sevstr, *subtype;
static const char *mc_ue_types[] = {
@ -387,7 +388,9 @@ void machine_check_print_event_info(struct machine_check_event *evt,
evt->disposition == MCE_DISPOSITION_RECOVERED ?
"Recovered" : "Not recovered");
if (user_mode) {
if (in_guest) {
printk("%s Guest NIP: %016llx\n", level, evt->srr0);
} else if (user_mode) {
printk("%s NIP: [%016llx] PID: %d Comm: %s\n", level,
evt->srr0, current->pid, current->comm);
} else {
@ -488,6 +491,8 @@ long machine_check_early(struct pt_regs *regs)
{
long handled = 0;
hv_nmi_check_nonrecoverable(regs);
/*
* See if platform is capable of handling machine check.
*/

17
arch/powerpc/kernel/misc_32.S
View File

@ -46,11 +46,10 @@ _GLOBAL(call_do_softirq)
mflr r0
stw r0,4(r1)
lwz r10,THREAD+KSP_LIMIT(r2)
addi r11,r3,THREAD_INFO_GAP
stw r3, THREAD+KSP_LIMIT(r2)
stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
mr r1,r3
stw r10,8(r1)
stw r11,THREAD+KSP_LIMIT(r2)
bl __do_softirq
lwz r10,8(r1)
lwz r1,0(r1)
@ -60,17 +59,16 @@ _GLOBAL(call_do_softirq)
blr
/*
* void call_do_irq(struct pt_regs *regs, struct thread_info *irqtp);
* void call_do_irq(struct pt_regs *regs, void *sp);
*/
_GLOBAL(call_do_irq)
mflr r0
stw r0,4(r1)
lwz r10,THREAD+KSP_LIMIT(r2)
addi r11,r4,THREAD_INFO_GAP
stw r4, THREAD+KSP_LIMIT(r2)
stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
mr r1,r4
stw r10,8(r1)
stw r11,THREAD+KSP_LIMIT(r2)
bl __do_irq
lwz r10,8(r1)
lwz r1,0(r1)
@ -183,10 +181,13 @@ _GLOBAL(low_choose_750fx_pll)
or r4,r4,r5
mtspr SPRN_HID1,r4
#ifdef CONFIG_SMP
/* Store new HID1 image */
CURRENT_THREAD_INFO(r6, r1)
lwz r6,TI_CPU(r6)
lwz r6,TASK_CPU(r2)
slwi r6,r6,2
#else
li r6, 0
#endif
addis r6,r6,nap_save_hid1@ha
stw r4,nap_save_hid1@l(r6)
@ -599,7 +600,7 @@ EXPORT_SYMBOL(__bswapdi2)
#ifdef CONFIG_SMP
_GLOBAL(start_secondary_resume)
/* Reset stack */
CURRENT_THREAD_INFO(r1, r1)
rlwinm r1, r1, 0, 0, 31 - THREAD_SHIFT
addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
li r3,0
stw r3,0(r1) /* Zero the stack frame pointer */

21
arch/powerpc/kernel/pci-common.c
View File

@ -63,19 +63,13 @@ resource_size_t isa_mem_base;
EXPORT_SYMBOL(isa_mem_base);
static const struct dma_map_ops *pci_dma_ops = &dma_nommu_ops;
static const struct dma_map_ops *pci_dma_ops;
void set_pci_dma_ops(const struct dma_map_ops *dma_ops)
{
pci_dma_ops = dma_ops;
}
const struct dma_map_ops *get_pci_dma_ops(void)
{
return pci_dma_ops;
}
EXPORT_SYMBOL(get_pci_dma_ops);
/*
* This function should run under locking protection, specifically
* hose_spinlock.
@ -358,6 +352,17 @@ struct pci_controller* pci_find_hose_for_OF_device(struct device_node* node)
return NULL;
}
struct pci_controller *pci_find_controller_for_domain(int domain_nr)
{
struct pci_controller *hose;
list_for_each_entry(hose, &hose_list, list_node)
if (hose->global_number == domain_nr)
return hose;
return NULL;
}
/*
* Reads the interrupt pin to determine if interrupt is use by card.
* If the interrupt is used, then gets the interrupt line from the
@ -973,7 +978,7 @@ static void pcibios_setup_device(struct pci_dev *dev)
/* Hook up default DMA ops */
set_dma_ops(&dev->dev, pci_dma_ops);
set_dma_offset(&dev->dev, PCI_DRAM_OFFSET);
dev->dev.archdata.dma_offset = PCI_DRAM_OFFSET;
/* Additional platform DMA/iommu setup */
phb = pci_bus_to_host(dev->bus);

66
arch/powerpc/kernel/process.c
View File

@ -176,7 +176,7 @@ static void __giveup_fpu(struct task_struct *tsk)
save_fpu(tsk);
msr = tsk->thread.regs->msr;
msr &= ~MSR_FP;
msr &= ~(MSR_FP|MSR_FE0|MSR_FE1);
#ifdef CONFIG_VSX
if (cpu_has_feature(CPU_FTR_VSX))
msr &= ~MSR_VSX;
@ -1231,8 +1231,8 @@ struct task_struct *__switch_to(struct task_struct *prev,
batch->active = 1;
}
if (current_thread_info()->task->thread.regs) {
restore_math(current_thread_info()->task->thread.regs);
if (current->thread.regs) {
restore_math(current->thread.regs);
/*
* The copy-paste buffer can only store into foreign real
@ -1242,7 +1242,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
* mappings, we must issue a cp_abort to clear any state and
* prevent snooping, corruption or a covert channel.
*/
if (current_thread_info()->task->thread.used_vas)
if (current->thread.used_vas)
asm volatile(PPC_CP_ABORT);
}
#endif /* CONFIG_PPC_BOOK3S_64 */
@ -1634,7 +1634,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
unsigned long sp = (unsigned long)task_stack_page(p) + THREAD_SIZE;
struct thread_info *ti = task_thread_info(p);
klp_init_thread_info(ti);
klp_init_thread_info(p);
/* Copy registers */
sp -= sizeof(struct pt_regs);
@ -1691,8 +1691,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
sp -= STACK_FRAME_OVERHEAD;
p->thread.ksp = sp;
#ifdef CONFIG_PPC32
p->thread.ksp_limit = (unsigned long)task_stack_page(p) +
_ALIGN_UP(sizeof(struct thread_info), 16);
p->thread.ksp_limit = (unsigned long)end_of_stack(p);
#endif
#ifdef CONFIG_HAVE_HW_BREAKPOINT
p->thread.ptrace_bps[0] = NULL;
@ -1995,21 +1994,14 @@ static inline int valid_irq_stack(unsigned long sp, struct task_struct *p,
unsigned long stack_page;
unsigned long cpu = task_cpu(p);
/*
* Avoid crashing if the stack has overflowed and corrupted
* task_cpu(p), which is in the thread_info struct.
*/
if (cpu < NR_CPUS && cpu_possible(cpu)) {
stack_page = (unsigned long) hardirq_ctx[cpu];
if (sp >= stack_page + sizeof(struct thread_struct)
&& sp <= stack_page + THREAD_SIZE - nbytes)
return 1;
stack_page = (unsigned long)hardirq_ctx[cpu];
if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
return 1;
stack_page = (unsigned long)softirq_ctx[cpu];
if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
return 1;
stack_page = (unsigned long) softirq_ctx[cpu];
if (sp >= stack_page + sizeof(struct thread_struct)
&& sp <= stack_page + THREAD_SIZE - nbytes)
return 1;
}
return 0;
}
@ -2018,8 +2010,10 @@ int validate_sp(unsigned long sp, struct task_struct *p,
{
unsigned long stack_page = (unsigned long)task_stack_page(p);
if (sp >= stack_page + sizeof(struct thread_struct)
&& sp <= stack_page + THREAD_SIZE - nbytes)
if (sp < THREAD_SIZE)
return 0;
if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
return 1;
return valid_irq_stack(sp, p, nbytes);
@ -2027,7 +2021,7 @@ int validate_sp(unsigned long sp, struct task_struct *p,
EXPORT_SYMBOL(validate_sp);
unsigned long get_wchan(struct task_struct *p)
static unsigned long __get_wchan(struct task_struct *p)
{
unsigned long ip, sp;
int count = 0;
@ -2053,6 +2047,20 @@ unsigned long get_wchan(struct task_struct *p)
return 0;
}
unsigned long get_wchan(struct task_struct *p)
{
unsigned long ret;
if (!try_get_task_stack(p))
return 0;
ret = __get_wchan(p);
put_task_stack(p);
return ret;
}
static int kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH;
void show_stack(struct task_struct *tsk, unsigned long *stack)
@ -2067,9 +2075,13 @@ void show_stack(struct task_struct *tsk, unsigned long *stack)
int curr_frame = 0;
#endif
sp = (unsigned long) stack;
if (tsk == NULL)
tsk = current;
if (!try_get_task_stack(tsk))
return;
sp = (unsigned long) stack;
if (sp == 0) {
if (tsk == current)
sp = current_stack_pointer();
@ -2081,7 +2093,7 @@ void show_stack(struct task_struct *tsk, unsigned long *stack)
printk("Call Trace:\n");
do {
if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
return;
break;
stack = (unsigned long *) sp;
newsp = stack[0];
@ -2121,6 +2133,8 @@ void show_stack(struct task_struct *tsk, unsigned long *stack)
sp = newsp;
} while (count++ < kstack_depth_to_print);
put_task_stack(tsk);
}
#ifdef CONFIG_PPC64

18
arch/powerpc/kernel/ptrace.c
View File

@ -33,6 +33,7 @@
#include <linux/hw_breakpoint.h>
#include <linux/perf_event.h>
#include <linux/context_tracking.h>
#include <linux/nospec.h>
#include <linux/uaccess.h>
#include <linux/pkeys.h>
@ -274,6 +275,8 @@ static int set_user_trap(struct task_struct *task, unsigned long trap)
*/
int ptrace_get_reg(struct task_struct *task, int regno, unsigned long *data)
{
unsigned int regs_max;
if ((task->thread.regs == NULL) || !data)
return -EIO;
@ -297,7 +300,9 @@ int ptrace_get_reg(struct task_struct *task, int regno, unsigned long *data)
}
#endif
if (regno < (sizeof(struct user_pt_regs) / sizeof(unsigned long))) {
regs_max = sizeof(struct user_pt_regs) / sizeof(unsigned long);
if (regno < regs_max) {
regno = array_index_nospec(regno, regs_max);
*data = ((unsigned long *)task->thread.regs)[regno];
return 0;
}
@ -321,6 +326,7 @@ int ptrace_put_reg(struct task_struct *task, int regno, unsigned long data)
return set_user_dscr(task, data);
if (regno <= PT_MAX_PUT_REG) {
regno = array_index_nospec(regno, PT_MAX_PUT_REG + 1);
((unsigned long *)task->thread.regs)[regno] = data;
return 0;
}
@ -561,6 +567,7 @@ static int vr_get(struct task_struct *target, const struct user_regset *regset,
/*
* Copy out only the low-order word of vrsave.
*/
int start, end;
union {
elf_vrreg_t reg;
u32 word;
@ -569,8 +576,10 @@ static int vr_get(struct task_struct *target, const struct user_regset *regset,
vrsave.word = target->thread.vrsave;
start = 33 * sizeof(vector128);
end = start + sizeof(vrsave);
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
start, end);
}
return ret;
@ -608,6 +617,7 @@ static int vr_set(struct task_struct *target, const struct user_regset *regset,
/*
* We use only the first word of vrsave.
*/
int start, end;
union {
elf_vrreg_t reg;
u32 word;
@ -616,8 +626,10 @@ static int vr_set(struct task_struct *target, const struct user_regset *regset,
vrsave.word = target->thread.vrsave;
start = 33 * sizeof(vector128);
end = start + sizeof(vrsave);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
start, end);
if (!ret)
target->thread.vrsave = vrsave.word;
}

5
arch/powerpc/kernel/setup-common.c
View File

@ -634,7 +634,7 @@ void probe_machine(void)
}
/* What can we do if we didn't find ? */
if (machine_id >= &__machine_desc_end) {
DBG("No suitable machine found !\n");
pr_err("No suitable machine description found !\n");
for (;;);
}
@ -791,7 +791,6 @@ void arch_setup_pdev_archdata(struct platform_device *pdev)
{
pdev->archdata.dma_mask = DMA_BIT_MASK(32);
pdev->dev.dma_mask = &pdev->archdata.dma_mask;
set_dma_ops(&pdev->dev, &dma_nommu_ops);
}
static __init void print_system_info(void)
@ -938,7 +937,7 @@ void __init setup_arch(char **cmdline_p)
/* Reserve large chunks of memory for use by CMA for KVM. */
kvm_cma_reserve();
klp_init_thread_info(&init_thread_info);
klp_init_thread_info(&init_task);
init_mm.start_code = (unsigned long)_stext;
init_mm.end_code = (unsigned long) _etext;

26
arch/powerpc/kernel/setup_32.c
View File

@ -162,6 +162,17 @@ static int __init ppc_init(void)
}
arch_initcall(ppc_init);
static void *__init alloc_stack(void)
{
void *ptr = memblock_alloc(THREAD_SIZE, THREAD_SIZE);
if (!ptr)
panic("cannot allocate %d bytes for stack at %pS\n",
THREAD_SIZE, (void *)_RET_IP_);
return ptr;
}
void __init irqstack_early_init(void)
{
unsigned int i;
@ -169,10 +180,8 @@ void __init irqstack_early_init(void)
/* interrupt stacks must be in lowmem, we get that for free on ppc32
* as the memblock is limited to lowmem by default */
for_each_possible_cpu(i) {
softirq_ctx[i] = (struct thread_info *)
__va(memblock_phys_alloc(THREAD_SIZE, THREAD_SIZE));
hardirq_ctx[i] = (struct thread_info *)
__va(memblock_phys_alloc(THREAD_SIZE, THREAD_SIZE));
softirq_ctx[i] = alloc_stack();
hardirq_ctx[i] = alloc_stack();
}
}
@ -190,13 +199,10 @@ void __init exc_lvl_early_init(void)
hw_cpu = 0;
#endif
critirq_ctx[hw_cpu] = (struct thread_info *)
__va(memblock_phys_alloc(THREAD_SIZE, THREAD_SIZE));
critirq_ctx[hw_cpu] = alloc_stack();
#ifdef CONFIG_BOOKE
dbgirq_ctx[hw_cpu] = (struct thread_info *)
__va(memblock_phys_alloc(THREAD_SIZE, THREAD_SIZE));
mcheckirq_ctx[hw_cpu] = (struct thread_info *)
__va(memblock_phys_alloc(THREAD_SIZE, THREAD_SIZE));
dbgirq_ctx[hw_cpu] = alloc_stack();
mcheckirq_ctx[hw_cpu] = alloc_stack();
#endif
}
}

51
arch/powerpc/kernel/setup_64.c
View File

@ -634,19 +634,17 @@ __init u64 ppc64_bolted_size(void)
static void *__init alloc_stack(unsigned long limit, int cpu)
{
unsigned long pa;
void *ptr;
BUILD_BUG_ON(STACK_INT_FRAME_SIZE % 16);
pa = memblock_alloc_base_nid(THREAD_SIZE, THREAD_SIZE, limit,
early_cpu_to_node(cpu), MEMBLOCK_NONE);
if (!pa) {
pa = memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit);
if (!pa)
panic("cannot allocate stacks");
}
ptr = memblock_alloc_try_nid(THREAD_SIZE, THREAD_SIZE,
MEMBLOCK_LOW_LIMIT, limit,
early_cpu_to_node(cpu));
if (!ptr)
panic("cannot allocate stacks");
return __va(pa);
return ptr;
}
void __init irqstack_early_init(void)
@ -691,24 +689,6 @@ void __init exc_lvl_early_init(void)
}
#endif
/*
* Emergency stacks are used for a range of things, from asynchronous
* NMIs (system reset, machine check) to synchronous, process context.
* We set preempt_count to zero, even though that isn't necessarily correct. To
* get the right value we'd need to copy it from the previous thread_info, but
* doing that might fault causing more problems.
* TODO: what to do with accounting?
*/
static void emerg_stack_init_thread_info(struct thread_info *ti, int cpu)
{
ti->task = NULL;
ti->cpu = cpu;
ti->preempt_count = 0;
ti->local_flags = 0;
ti->flags = 0;
klp_init_thread_info(ti);
}
/*
* Stack space used when we detect a bad kernel stack pointer, and
* early in SMP boots before relocation is enabled. Exclusive emergency
@ -736,25 +716,14 @@ void __init emergency_stack_init(void)
limit = min(ppc64_bolted_size(), ppc64_rma_size);
for_each_possible_cpu(i) {
struct thread_info *ti;
ti = alloc_stack(limit, i);
memset(ti, 0, THREAD_SIZE);
emerg_stack_init_thread_info(ti, i);
paca_ptrs[i]->emergency_sp = (void *)ti + THREAD_SIZE;
paca_ptrs[i]->emergency_sp = alloc_stack(limit, i) + THREAD_SIZE;
#ifdef CONFIG_PPC_BOOK3S_64
/* emergency stack for NMI exception handling. */
ti = alloc_stack(limit, i);
memset(ti, 0, THREAD_SIZE);
emerg_stack_init_thread_info(ti, i);
paca_ptrs[i]->nmi_emergency_sp = (void *)ti + THREAD_SIZE;
paca_ptrs[i]->nmi_emergency_sp = alloc_stack(limit, i) + THREAD_SIZE;
/* emergency stack for machine check exception handling. */
ti = alloc_stack(limit, i);
memset(ti, 0, THREAD_SIZE);
emerg_stack_init_thread_info(ti, i);
paca_ptrs[i]->mc_emergency_sp = (void *)ti + THREAD_SIZE;
paca_ptrs[i]->mc_emergency_sp = alloc_stack(limit, i) + THREAD_SIZE;
#endif
}
}

109
arch/powerpc/kernel/smp.c
View File

@ -20,6 +20,7 @@
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/sched/mm.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/topology.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
@ -75,7 +76,7 @@
static DEFINE_PER_CPU(int, cpu_state) = { 0 };
#endif
struct thread_info *secondary_ti;
struct task_struct *secondary_current;
bool has_big_cores;
DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
@ -358,13 +359,12 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask)
* NMI IPIs may not be recoverable, so should not be used as ongoing part of
* a running system. They can be used for crash, debug, halt/reboot, etc.
*
* NMI IPIs are globally single threaded. No more than one in progress at
* any time.
*
* The IPI call waits with interrupts disabled until all targets enter the
* NMI handler, then the call returns.
* NMI handler, then returns. Subsequent IPIs can be issued before targets
* have returned from their handlers, so there is no guarantee about
* concurrency or re-entrancy.
*
* No new NMI can be initiated until targets exit the handler.
* A new NMI can be issued before all targets exit the handler.
*
* The IPI call may time out without all targets entering the NMI handler.
* In that case, there is some logic to recover (and ignore subsequent
@ -375,7 +375,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask)
static atomic_t __nmi_ipi_lock = ATOMIC_INIT(0);
static struct cpumask nmi_ipi_pending_mask;
static int nmi_ipi_busy_count = 0;
static bool nmi_ipi_busy = false;
static void (*nmi_ipi_function)(struct pt_regs *) = NULL;
static void nmi_ipi_lock_start(unsigned long *flags)
@ -414,7 +414,7 @@ static void nmi_ipi_unlock_end(unsigned long *flags)
*/
int smp_handle_nmi_ipi(struct pt_regs *regs)
{
void (*fn)(struct pt_regs *);
void (*fn)(struct pt_regs *) = NULL;
unsigned long flags;
int me = raw_smp_processor_id();
int ret = 0;
@ -425,29 +425,17 @@ int smp_handle_nmi_ipi(struct pt_regs *regs)
* because the caller may have timed out.
*/
nmi_ipi_lock_start(&flags);
if (!nmi_ipi_busy_count)
goto out;
if (!cpumask_test_cpu(me, &nmi_ipi_pending_mask))
goto out;
fn = nmi_ipi_function;
if (!fn)
goto out;
cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
nmi_ipi_busy_count++;
nmi_ipi_unlock();
ret = 1;
fn(regs);
nmi_ipi_lock();
if (nmi_ipi_busy_count > 1) /* Can race with caller time-out */
nmi_ipi_busy_count--;
out:
if (cpumask_test_cpu(me, &nmi_ipi_pending_mask)) {
cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
fn = READ_ONCE(nmi_ipi_function);
WARN_ON_ONCE(!fn);
ret = 1;
}
nmi_ipi_unlock_end(&flags);
if (fn)
fn(regs);
return ret;
}
@ -473,9 +461,10 @@ static void do_smp_send_nmi_ipi(int cpu, bool safe)
* - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS.
* - fn is the target callback function.
* - delay_us > 0 is the delay before giving up waiting for targets to
* complete executing the handler, == 0 specifies indefinite delay.
* begin executing the handler, == 0 specifies indefinite delay.
*/
int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us, bool safe)
static int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *),
u64 delay_us, bool safe)
{
unsigned long flags;
int me = raw_smp_processor_id();
@ -487,31 +476,33 @@ int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us, bool
if (unlikely(!smp_ops))
return 0;
/* Take the nmi_ipi_busy count/lock with interrupts hard disabled */
nmi_ipi_lock_start(&flags);
while (nmi_ipi_busy_count) {
while (nmi_ipi_busy) {
nmi_ipi_unlock_end(&flags);
spin_until_cond(nmi_ipi_busy_count == 0);
spin_until_cond(!nmi_ipi_busy);
nmi_ipi_lock_start(&flags);
}
nmi_ipi_busy = true;
nmi_ipi_function = fn;
WARN_ON_ONCE(!cpumask_empty(&nmi_ipi_pending_mask));
if (cpu < 0) {
/* ALL_OTHERS */
cpumask_copy(&nmi_ipi_pending_mask, cpu_online_mask);
cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
} else {
/* cpumask starts clear */
cpumask_set_cpu(cpu, &nmi_ipi_pending_mask);
}
nmi_ipi_busy_count++;
nmi_ipi_unlock();
/* Interrupts remain hard disabled */
do_smp_send_nmi_ipi(cpu, safe);
nmi_ipi_lock();
/* nmi_ipi_busy_count is held here, so unlock/lock is okay */
/* nmi_ipi_busy is set here, so unlock/lock is okay */
while (!cpumask_empty(&nmi_ipi_pending_mask)) {
nmi_ipi_unlock();
udelay(1);
@ -523,29 +514,15 @@ int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us, bool
}
}
while (nmi_ipi_busy_count > 1) {
nmi_ipi_unlock();
udelay(1);
nmi_ipi_lock();
if (delay_us) {
delay_us--;
if (!delay_us)
break;
}
}
if (!cpumask_empty(&nmi_ipi_pending_mask)) {
/* Timeout waiting for CPUs to call smp_handle_nmi_ipi */
ret = 0;
cpumask_clear(&nmi_ipi_pending_mask);
}
if (nmi_ipi_busy_count > 1) {
/* Timeout waiting for CPUs to execute fn */
ret = 0;
nmi_ipi_busy_count = 1;
}
nmi_ipi_busy_count--;
nmi_ipi_function = NULL;
nmi_ipi_busy = false;
nmi_ipi_unlock_end(&flags);
return ret;
@ -613,17 +590,8 @@ void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
static void nmi_stop_this_cpu(struct pt_regs *regs)
{
/*
* This is a special case because it never returns, so the NMI IPI
* handling would never mark it as done, which makes any later
* smp_send_nmi_ipi() call spin forever. Mark it done now.
*
* IRQs are already hard disabled by the smp_handle_nmi_ipi.
*/
nmi_ipi_lock();
if (nmi_ipi_busy_count > 1)
nmi_ipi_busy_count--;
nmi_ipi_unlock();
spin_begin();
while (1)
spin_cpu_relax();
@ -663,7 +631,7 @@ void smp_send_stop(void)
}
#endif /* CONFIG_NMI_IPI */
struct thread_info *current_set[NR_CPUS];
struct task_struct *current_set[NR_CPUS];
static void smp_store_cpu_info(int id)
{
@ -928,7 +896,7 @@ void smp_prepare_boot_cpu(void)
paca_ptrs[boot_cpuid]->__current = current;
#endif
set_numa_node(numa_cpu_lookup_table[boot_cpuid]);
current_set[boot_cpuid] = task_thread_info(current);
current_set[boot_cpuid] = current;
}
#ifdef CONFIG_HOTPLUG_CPU
@ -1013,14 +981,13 @@ static bool secondaries_inhibited(void)
static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
{
struct thread_info *ti = task_thread_info(idle);
#ifdef CONFIG_PPC64
paca_ptrs[cpu]->__current = idle;
paca_ptrs[cpu]->kstack = (unsigned long)ti + THREAD_SIZE - STACK_FRAME_OVERHEAD;
paca_ptrs[cpu]->kstack = (unsigned long)task_stack_page(idle) +
THREAD_SIZE - STACK_FRAME_OVERHEAD;
#endif
ti->cpu = cpu;
secondary_ti = current_set[cpu] = ti;
idle->cpu = cpu;
secondary_current = current_set[cpu] = idle;
}
int __cpu_up(unsigned int cpu, struct task_struct *tidle)

104
arch/powerpc/kernel/stacktrace.c
View File

@ -67,12 +67,17 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
{
unsigned long sp;
if (!try_get_task_stack(tsk))
return;
if (tsk == current)
sp = current_stack_pointer();
else
sp = tsk->thread.ksp;
save_context_stack(trace, sp, tsk, 0);
put_task_stack(tsk);
}
EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
@ -84,25 +89,21 @@ save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
EXPORT_SYMBOL_GPL(save_stack_trace_regs);
#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE
int
save_stack_trace_tsk_reliable(struct task_struct *tsk,
struct stack_trace *trace)
/*
* This function returns an error if it detects any unreliable features of the
* stack. Otherwise it guarantees that the stack trace is reliable.
*
* If the task is not 'current', the caller *must* ensure the task is inactive.
*/
static int __save_stack_trace_tsk_reliable(struct task_struct *tsk,
struct stack_trace *trace)
{
unsigned long sp;
unsigned long newsp;
unsigned long stack_page = (unsigned long)task_stack_page(tsk);
unsigned long stack_end;
int graph_idx = 0;
/*
* The last frame (unwinding first) may not yet have saved
* its LR onto the stack.
*/
int firstframe = 1;
if (tsk == current)
sp = current_stack_pointer();
else
sp = tsk->thread.ksp;
bool firstframe;
stack_end = stack_page + THREAD_SIZE;
if (!is_idle_task(tsk)) {
@ -129,40 +130,53 @@ save_stack_trace_tsk_reliable(struct task_struct *tsk,
stack_end -= STACK_FRAME_OVERHEAD;
}
if (tsk == current)
sp = current_stack_pointer();
else
sp = tsk->thread.ksp;
if (sp < stack_page + sizeof(struct thread_struct) ||
sp > stack_end - STACK_FRAME_MIN_SIZE) {
return 1;
return -EINVAL;
}
for (;;) {
for (firstframe = true; sp != stack_end;
firstframe = false, sp = newsp) {
unsigned long *stack = (unsigned long *) sp;
unsigned long newsp, ip;
unsigned long ip;
/* sanity check: ABI requires SP to be aligned 16 bytes. */
if (sp & 0xF)
return 1;
/* Mark stacktraces with exception frames as unreliable. */
if (sp <= stack_end - STACK_INT_FRAME_SIZE &&
stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
return 1;
}
return -EINVAL;
newsp = stack[0];
/* Stack grows downwards; unwinder may only go up. */
if (newsp <= sp)
return 1;
return -EINVAL;
if (newsp != stack_end &&
newsp > stack_end - STACK_FRAME_MIN_SIZE) {
return 1; /* invalid backlink, too far up. */
return -EINVAL; /* invalid backlink, too far up. */
}
/*
* We can only trust the bottom frame's backlink, the
* rest of the frame may be uninitialized, continue to
* the next.
*/
if (firstframe)
continue;
/* Mark stacktraces with exception frames as unreliable. */
if (sp <= stack_end - STACK_INT_FRAME_SIZE &&
stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
return -EINVAL;
}
/* Examine the saved LR: it must point into kernel code. */
ip = stack[STACK_FRAME_LR_SAVE];
if (!firstframe && !__kernel_text_address(ip))
return 1;
firstframe = 0;
if (!__kernel_text_address(ip))
return -EINVAL;
/*
* FIXME: IMHO these tests do not belong in
@ -175,25 +189,37 @@ save_stack_trace_tsk_reliable(struct task_struct *tsk,
* as unreliable.
*/
if (ip == (unsigned long)kretprobe_trampoline)
return 1;
return -EINVAL;
#endif
if (trace->nr_entries >= trace->max_entries)
return -E2BIG;
if (!trace->skip)
trace->entries[trace->nr_entries++] = ip;
else
trace->skip--;
if (newsp == stack_end)
break;
if (trace->nr_entries >= trace->max_entries)
return -E2BIG;
sp = newsp;
}
return 0;
}
EXPORT_SYMBOL_GPL(save_stack_trace_tsk_reliable);
int save_stack_trace_tsk_reliable(struct task_struct *tsk,
struct stack_trace *trace)
{
int ret;
/*
* If the task doesn't have a stack (e.g., a zombie), the stack is
* "reliably" empty.
*/
if (!try_get_task_stack(tsk))
return 0;
ret = __save_stack_trace_tsk_reliable(tsk, trace);
put_task_stack(tsk);
return ret;
}
#endif /* CONFIG_HAVE_RELIABLE_STACKTRACE */
#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_NMI_IPI)

2
arch/powerpc/kernel/syscalls.c
View File

@ -123,7 +123,7 @@ long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,
(u64)len_high << 32 | len_low, advice);
}
long sys_switch_endian(void)
SYSCALL_DEFINE0(switch_endian)
{
struct thread_info *ti;

4
arch/powerpc/kernel/syscalls/syscalltbl.sh
View File

@ -13,10 +13,10 @@ emit() {
t_entry="$3"
while [ $t_nxt -lt $t_nr ]; do
printf "__SYSCALL(%s,sys_ni_syscall, )\n" "${t_nxt}"
printf "__SYSCALL(%s,sys_ni_syscall)\n" "${t_nxt}"
t_nxt=$((t_nxt+1))
done
printf "__SYSCALL(%s,%s, )\n" "${t_nxt}" "${t_entry}"
printf "__SYSCALL(%s,%s)\n" "${t_nxt}" "${t_entry}"
}
grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | (

6
arch/powerpc/kernel/systbl.S
View File

@ -25,11 +25,11 @@
.globl sys_call_table
sys_call_table:
#ifdef CONFIG_PPC64
#define __SYSCALL(nr, entry, nargs) .8byte DOTSYM(entry)
#define __SYSCALL(nr, entry) .8byte DOTSYM(entry)
#include <asm/syscall_table_64.h>
#undef __SYSCALL
#else
#define __SYSCALL(nr, entry, nargs) .long entry
#define __SYSCALL(nr, entry) .long entry
#include <asm/syscall_table_32.h>
#undef __SYSCALL
#endif
@ -38,7 +38,7 @@ sys_call_table:
.globl compat_sys_call_table
compat_sys_call_table:
#define compat_sys_sigsuspend sys_sigsuspend
#define __SYSCALL(nr, entry, nargs) .8byte DOTSYM(entry)
#define __SYSCALL(nr, entry) .8byte DOTSYM(entry)
#include <asm/syscall_table_c32.h>
#undef __SYSCALL
#endif

1
arch/powerpc/kernel/time.c
View File

@ -57,7 +57,6 @@
#include <linux/irq_work.h>
#include <linux/clk-provider.h>
#include <linux/suspend.h>
#include <linux/rtc.h>
#include <linux/sched/cputime.h>
#include <linux/processor.h>
#include <asm/trace.h>

3
arch/powerpc/kernel/trace/Makefile
View File

@ -23,6 +23,7 @@ obj-$(CONFIG_TRACING) += trace_clock.o
obj-$(CONFIG_PPC64) += $(obj64-y)
obj-$(CONFIG_PPC32) += $(obj32-y)
# Disable GCOV & sanitizers in odd or sensitive code
# Disable GCOV, KCOV & sanitizers in odd or sensitive code
GCOV_PROFILE_ftrace.o := n
KCOV_INSTRUMENT_ftrace.o := n
UBSAN_SANITIZE_ftrace.o := n

Some files were not shown because too many files have changed in this diff Show More