You've already forked linux-apfs
mirror of
https://github.com/linux-apfs/linux-apfs.git
synced 2026-05-01 15:00:59 -07:00
Merge tag 'powerpc-4.7-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux
Pull powerpc updates from Michael Ellerman:
"Highlights:
- Support for Power ISA 3.0 (Power9) Radix Tree MMU from Aneesh Kumar K.V
- Live patching support for ppc64le (also merged via livepatching.git)
Various cleanups & minor fixes from:
- Aaro Koskinen, Alexey Kardashevskiy, Andrew Donnellan, Aneesh Kumar K.V,
Chris Smart, Daniel Axtens, Frederic Barrat, Gavin Shan, Ian Munsie,
Lennart Sorensen, Madhavan Srinivasan, Mahesh Salgaonkar, Markus Elfring,
Michael Ellerman, Oliver O'Halloran, Paul Gortmaker, Paul Mackerras,
Rashmica Gupta, Russell Currey, Suraj Jitindar Singh, Thiago Jung
Bauermann, Valentin Rothberg, Vipin K Parashar.
General:
- Update LMB associativity index during DLPAR add/remove from Nathan
Fontenot
- Fix branching to OOL handlers in relocatable kernel from Hari Bathini
- Add support for userspace Power9 copy/paste from Chris Smart
- Always use STRICT_MM_TYPECHECKS from Michael Ellerman
- Add mask of possible MMU features from Michael Ellerman
PCI:
- Enable pass through of NVLink to guests from Alexey Kardashevskiy
- Cleanups in preparation for powernv PCI hotplug from Gavin Shan
- Don't report error in eeh_pe_reset_and_recover() from Gavin Shan
- Restore initial state in eeh_pe_reset_and_recover() from Gavin Shan
- Revert "powerpc/eeh: Fix crash in eeh_add_device_early() on Cell"
from Guilherme G Piccoli
- Remove the dependency on EEH struct in DDW mechanism from Guilherme
G Piccoli
selftests:
- Test cp_abort during context switch from Chris Smart
- Add several tests for transactional memory support from Rashmica
Gupta
perf:
- Add support for sampling interrupt register state from Anju T
- Add support for unwinding perf-stackdump from Chandan Kumar
cxl:
- Configure the PSL for two CAPI ports on POWER8NVL from Philippe
Bergheaud
- Allow initialization on timebase sync failures from Frederic Barrat
- Increase timeout for detection of AFU mmio hang from Frederic
Barrat
- Handle num_of_processes larger than can fit in the SPA from Ian
Munsie
- Ensure PSL interrupt is configured for contexts with no AFU IRQs
from Ian Munsie
- Add kernel API to allow a context to operate with relocate disabled
from Ian Munsie
- Check periodically the coherent platform function's state from
Christophe Lombard
Freescale:
- Updates from Scott: "Contains 86xx fixes, minor device tree fixes,
an erratum workaround, and a kconfig dependency fix."
* tag 'powerpc-4.7-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux: (192 commits)
powerpc/86xx: Fix PCI interrupt map definition
powerpc/86xx: Move pci1 definition to the include file
powerpc/fsl: Fix build of the dtb embedded kernel images
powerpc/fsl: Fix rcpm compatible string
powerpc/fsl: Remove FSL_SOC dependency from FSL_LBC
powerpc/fsl-pci: Add a workaround for PCI 5 errata
powerpc/fsl: Fix SPI compatible on t208xrdb and t1040rdb
powerpc/powernv/npu: Add PE to PHB's list
powerpc/powernv: Fix insufficient memory allocation
powerpc/iommu: Remove the dependency on EEH struct in DDW mechanism
Revert "powerpc/eeh: Fix crash in eeh_add_device_early() on Cell"
powerpc/eeh: Drop unnecessary label in eeh_pe_change_owner()
powerpc/eeh: Ignore handlers in eeh_pe_reset_and_recover()
powerpc/eeh: Restore initial state in eeh_pe_reset_and_recover()
powerpc/eeh: Don't report error in eeh_pe_reset_and_recover()
Revert "powerpc/powernv: Exclude root bus in pnv_pci_reset_secondary_bus()"
powerpc/powernv/npu: Enable NVLink pass through
powerpc/powernv/npu: Rework TCE Kill handling
powerpc/powernv/npu: Add set/unset window helpers
powerpc/powernv/ioda2: Export debug helper pe_level_printk()
...
This commit is contained in:
Linus Torvalds
@@ -39,8 +39,5 @@
|
||||
#define _PMD_PRESENT_MASK (PAGE_MASK)
|
||||
#define _PMD_BAD (~PAGE_MASK)
|
||||
|
||||
/* Hash table based platforms need atomic updates of the linux PTE */
|
||||
#define PTE_ATOMIC_UPDATES 1
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
#endif /* _ASM_POWERPC_BOOK3S_32_HASH_H */
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#ifndef _ASM_POWERPC_MMU_HASH32_H_
|
||||
#define _ASM_POWERPC_MMU_HASH32_H_
|
||||
#ifndef _ASM_POWERPC_BOOK3S_32_MMU_HASH_H_
|
||||
#define _ASM_POWERPC_BOOK3S_32_MMU_HASH_H_
|
||||
/*
|
||||
* 32-bit hash table MMU support
|
||||
*/
|
||||
@@ -90,4 +90,4 @@ typedef struct {
|
||||
#define mmu_virtual_psize MMU_PAGE_4K
|
||||
#define mmu_linear_psize MMU_PAGE_256M
|
||||
|
||||
#endif /* _ASM_POWERPC_MMU_HASH32_H_ */
|
||||
#endif /* _ASM_POWERPC_BOOK3S_32_MMU_HASH_H_ */
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_32_PGALLOC_H
|
||||
#define _ASM_POWERPC_BOOK3S_32_PGALLOC_H
|
||||
|
||||
#include <linux/threads.h>
|
||||
|
||||
/* For 32-bit, all levels of page tables are just drawn from get_free_page() */
|
||||
#define MAX_PGTABLE_INDEX_SIZE 0
|
||||
|
||||
extern void __bad_pte(pmd_t *pmd);
|
||||
|
||||
extern pgd_t *pgd_alloc(struct mm_struct *mm);
|
||||
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
|
||||
|
||||
/*
|
||||
* We don't have any real pmd's, and this code never triggers because
|
||||
* the pgd will always be present..
|
||||
*/
|
||||
/* #define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) */
|
||||
#define pmd_free(mm, x) do { } while (0)
|
||||
#define __pmd_free_tlb(tlb,x,a) do { } while (0)
|
||||
/* #define pgd_populate(mm, pmd, pte) BUG() */
|
||||
|
||||
#ifndef CONFIG_BOOKE
|
||||
|
||||
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pte_t *pte)
|
||||
{
|
||||
*pmdp = __pmd(__pa(pte) | _PMD_PRESENT);
|
||||
}
|
||||
|
||||
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pgtable_t pte_page)
|
||||
{
|
||||
*pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_PRESENT);
|
||||
}
|
||||
|
||||
#define pmd_pgtable(pmd) pmd_page(pmd)
|
||||
#else
|
||||
|
||||
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pte_t *pte)
|
||||
{
|
||||
*pmdp = __pmd((unsigned long)pte | _PMD_PRESENT);
|
||||
}
|
||||
|
||||
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pgtable_t pte_page)
|
||||
{
|
||||
*pmdp = __pmd((unsigned long)lowmem_page_address(pte_page) | _PMD_PRESENT);
|
||||
}
|
||||
|
||||
#define pmd_pgtable(pmd) pmd_page(pmd)
|
||||
#endif
|
||||
|
||||
extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr);
|
||||
extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr);
|
||||
|
||||
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
|
||||
{
|
||||
free_page((unsigned long)pte);
|
||||
}
|
||||
|
||||
static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
|
||||
{
|
||||
pgtable_page_dtor(ptepage);
|
||||
__free_page(ptepage);
|
||||
}
|
||||
|
||||
static inline void pgtable_free(void *table, unsigned index_size)
|
||||
{
|
||||
BUG_ON(index_size); /* 32-bit doesn't use this */
|
||||
free_page((unsigned long)table);
|
||||
}
|
||||
|
||||
#define check_pgt_cache() do { } while (0)
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
static inline void pgtable_free_tlb(struct mmu_gather *tlb,
|
||||
void *table, int shift)
|
||||
{
|
||||
unsigned long pgf = (unsigned long)table;
|
||||
BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
|
||||
pgf |= shift;
|
||||
tlb_remove_table(tlb, (void *)pgf);
|
||||
}
|
||||
|
||||
static inline void __tlb_remove_table(void *_table)
|
||||
{
|
||||
void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
|
||||
unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
|
||||
|
||||
pgtable_free(table, shift);
|
||||
}
|
||||
#else
|
||||
static inline void pgtable_free_tlb(struct mmu_gather *tlb,
|
||||
void *table, int shift)
|
||||
{
|
||||
pgtable_free(table, shift);
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
|
||||
unsigned long address)
|
||||
{
|
||||
tlb_flush_pgtable(tlb, address);
|
||||
pgtable_page_dtor(table);
|
||||
pgtable_free_tlb(tlb, page_address(table), 0);
|
||||
}
|
||||
#endif /* _ASM_POWERPC_BOOK3S_32_PGALLOC_H */
|
||||
@@ -5,58 +5,31 @@
|
||||
* for each page table entry. The PMD and PGD level use a 32b record for
|
||||
* each entry by assuming that each entry is page aligned.
|
||||
*/
|
||||
#define PTE_INDEX_SIZE 9
|
||||
#define PMD_INDEX_SIZE 7
|
||||
#define PUD_INDEX_SIZE 9
|
||||
#define PGD_INDEX_SIZE 9
|
||||
#define H_PTE_INDEX_SIZE 9
|
||||
#define H_PMD_INDEX_SIZE 7
|
||||
#define H_PUD_INDEX_SIZE 9
|
||||
#define H_PGD_INDEX_SIZE 9
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
#define PTE_TABLE_SIZE (sizeof(pte_t) << PTE_INDEX_SIZE)
|
||||
#define PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
|
||||
#define PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
|
||||
#define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
|
||||
#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
|
||||
#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
|
||||
#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
|
||||
|
||||
/* PMD_SHIFT determines what a second-level page table entry can map */
|
||||
#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
|
||||
#define PMD_SIZE (1UL << PMD_SHIFT)
|
||||
#define PMD_MASK (~(PMD_SIZE-1))
|
||||
#define H_PTE_TABLE_SIZE (sizeof(pte_t) << H_PTE_INDEX_SIZE)
|
||||
#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << H_PMD_INDEX_SIZE)
|
||||
#define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE)
|
||||
#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE)
|
||||
|
||||
/* With 4k base page size, hugepage PTEs go at the PMD level */
|
||||
#define MIN_HUGEPTE_SHIFT PMD_SHIFT
|
||||
|
||||
/* PUD_SHIFT determines what a third-level page table entry can map */
|
||||
#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
|
||||
#define PUD_SIZE (1UL << PUD_SHIFT)
|
||||
#define PUD_MASK (~(PUD_SIZE-1))
|
||||
|
||||
/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
|
||||
#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
|
||||
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
|
||||
#define PGDIR_MASK (~(PGDIR_SIZE-1))
|
||||
|
||||
/* Bits to mask out from a PMD to get to the PTE page */
|
||||
#define PMD_MASKED_BITS 0
|
||||
/* Bits to mask out from a PUD to get to the PMD page */
|
||||
#define PUD_MASKED_BITS 0
|
||||
/* Bits to mask out from a PGD to get to the PUD page */
|
||||
#define PGD_MASKED_BITS 0
|
||||
|
||||
/* PTE flags to conserve for HPTE identification */
|
||||
#define _PAGE_HPTEFLAGS (_PAGE_BUSY | _PAGE_HASHPTE | \
|
||||
_PAGE_F_SECOND | _PAGE_F_GIX)
|
||||
|
||||
/* shift to put page number into pte */
|
||||
#define PTE_RPN_SHIFT (12)
|
||||
#define PTE_RPN_SIZE (45) /* gives 57-bit real addresses */
|
||||
|
||||
#define _PAGE_4K_PFN 0
|
||||
#ifndef __ASSEMBLY__
|
||||
#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \
|
||||
H_PAGE_F_SECOND | H_PAGE_F_GIX)
|
||||
/*
|
||||
* Not supported by 4k linux page size
|
||||
*/
|
||||
#define H_PAGE_4K_PFN 0x0
|
||||
#define H_PAGE_THP_HUGE 0x0
|
||||
#define H_PAGE_COMBO 0x0
|
||||
#define H_PTE_FRAG_NR 0
|
||||
#define H_PTE_FRAG_SIZE_SHIFT 0
|
||||
/*
|
||||
* On all 4K setups, remap_4k_pfn() equates to remap_pfn_range()
|
||||
*/
|
||||
@@ -64,26 +37,7 @@
|
||||
remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, (prot))
|
||||
|
||||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
/*
|
||||
* For 4k page size, we support explicit hugepage via hugepd
|
||||
*/
|
||||
static inline int pmd_huge(pmd_t pmd)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int pud_huge(pud_t pud)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int pgd_huge(pgd_t pgd)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#define pgd_huge pgd_huge
|
||||
|
||||
static inline int hugepd_ok(hugepd_t hpd)
|
||||
static inline int hash__hugepd_ok(hugepd_t hpd)
|
||||
{
|
||||
/*
|
||||
* if it is not a pte and have hugepd shift mask
|
||||
@@ -94,7 +48,65 @@ static inline int hugepd_ok(hugepd_t hpd)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
#define is_hugepd(hpd) (hugepd_ok(hpd))
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
|
||||
static inline char *get_hpte_slot_array(pmd_t *pmdp)
|
||||
{
|
||||
BUG();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index)
|
||||
{
|
||||
BUG();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array,
|
||||
int index)
|
||||
{
|
||||
BUG();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array,
|
||||
unsigned int index, unsigned int hidx)
|
||||
{
|
||||
BUG();
|
||||
}
|
||||
|
||||
static inline int hash__pmd_trans_huge(pmd_t pmd)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b)
|
||||
{
|
||||
BUG();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline pmd_t hash__pmd_mkhuge(pmd_t pmd)
|
||||
{
|
||||
BUG();
|
||||
return pmd;
|
||||
}
|
||||
|
||||
extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp,
|
||||
unsigned long clr, unsigned long set);
|
||||
extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp);
|
||||
extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pgtable_t pgtable);
|
||||
extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
|
||||
extern void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp);
|
||||
extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp);
|
||||
extern int hash__has_transparent_hugepage(void);
|
||||
#endif
|
||||
|
||||
#endif /* !__ASSEMBLY__ */
|
||||
|
||||
@@ -1,73 +1,44 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H
|
||||
#define _ASM_POWERPC_BOOK3S_64_HASH_64K_H
|
||||
|
||||
#define PTE_INDEX_SIZE 8
|
||||
#define PMD_INDEX_SIZE 5
|
||||
#define PUD_INDEX_SIZE 5
|
||||
#define PGD_INDEX_SIZE 12
|
||||
|
||||
#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
|
||||
#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
|
||||
#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
|
||||
#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
|
||||
#define H_PTE_INDEX_SIZE 8
|
||||
#define H_PMD_INDEX_SIZE 5
|
||||
#define H_PUD_INDEX_SIZE 5
|
||||
#define H_PGD_INDEX_SIZE 12
|
||||
|
||||
/* With 4k base page size, hugepage PTEs go at the PMD level */
|
||||
#define MIN_HUGEPTE_SHIFT PAGE_SHIFT
|
||||
|
||||
/* PMD_SHIFT determines what a second-level page table entry can map */
|
||||
#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
|
||||
#define PMD_SIZE (1UL << PMD_SHIFT)
|
||||
#define PMD_MASK (~(PMD_SIZE-1))
|
||||
|
||||
/* PUD_SHIFT determines what a third-level page table entry can map */
|
||||
#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
|
||||
#define PUD_SIZE (1UL << PUD_SHIFT)
|
||||
#define PUD_MASK (~(PUD_SIZE-1))
|
||||
|
||||
/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
|
||||
#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
|
||||
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
|
||||
#define PGDIR_MASK (~(PGDIR_SIZE-1))
|
||||
|
||||
#define _PAGE_COMBO 0x00001000 /* this is a combo 4k page */
|
||||
#define _PAGE_4K_PFN 0x00002000 /* PFN is for a single 4k page */
|
||||
#define H_PAGE_COMBO 0x00001000 /* this is a combo 4k page */
|
||||
#define H_PAGE_4K_PFN 0x00002000 /* PFN is for a single 4k page */
|
||||
/*
|
||||
* Used to track subpage group valid if _PAGE_COMBO is set
|
||||
* This overloads _PAGE_F_GIX and _PAGE_F_SECOND
|
||||
* We need to differentiate between explicit huge page and THP huge
|
||||
* page, since THP huge page also need to track real subpage details
|
||||
*/
|
||||
#define _PAGE_COMBO_VALID (_PAGE_F_GIX | _PAGE_F_SECOND)
|
||||
#define H_PAGE_THP_HUGE H_PAGE_4K_PFN
|
||||
|
||||
/*
|
||||
* Used to track subpage group valid if H_PAGE_COMBO is set
|
||||
* This overloads H_PAGE_F_GIX and H_PAGE_F_SECOND
|
||||
*/
|
||||
#define H_PAGE_COMBO_VALID (H_PAGE_F_GIX | H_PAGE_F_SECOND)
|
||||
|
||||
/* PTE flags to conserve for HPTE identification */
|
||||
#define _PAGE_HPTEFLAGS (_PAGE_BUSY | _PAGE_F_SECOND | \
|
||||
_PAGE_F_GIX | _PAGE_HASHPTE | _PAGE_COMBO)
|
||||
|
||||
/* Shift to put page number into pte.
|
||||
*
|
||||
* That gives us a max RPN of 41 bits, which means a max of 57 bits
|
||||
* of addressable physical space, or 53 bits for the special 4k PFNs.
|
||||
*/
|
||||
#define PTE_RPN_SHIFT (16)
|
||||
#define PTE_RPN_SIZE (41)
|
||||
|
||||
#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_F_SECOND | \
|
||||
H_PAGE_F_GIX | H_PAGE_HASHPTE | H_PAGE_COMBO)
|
||||
/*
|
||||
* we support 16 fragments per PTE page of 64K size.
|
||||
*/
|
||||
#define PTE_FRAG_NR 16
|
||||
#define H_PTE_FRAG_NR 16
|
||||
/*
|
||||
* We use a 2K PTE page fragment and another 2K for storing
|
||||
* real_pte_t hash index
|
||||
*/
|
||||
#define PTE_FRAG_SIZE_SHIFT 12
|
||||
#define H_PTE_FRAG_SIZE_SHIFT 12
|
||||
#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
|
||||
|
||||
/* Bits to mask out from a PMD to get to the PTE page */
|
||||
#define PMD_MASKED_BITS 0xc0000000000000ffUL
|
||||
/* Bits to mask out from a PUD to get to the PMD page */
|
||||
#define PUD_MASKED_BITS 0xc0000000000000ffUL
|
||||
/* Bits to mask out from a PGD to get to the PUD page */
|
||||
#define PGD_MASKED_BITS 0xc0000000000000ffUL
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
#include <asm/errno.h>
|
||||
|
||||
/*
|
||||
* With 64K pages on hash table, we have a special PTE format that
|
||||
@@ -83,9 +54,9 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
|
||||
|
||||
rpte.pte = pte;
|
||||
rpte.hidx = 0;
|
||||
if (pte_val(pte) & _PAGE_COMBO) {
|
||||
if (pte_val(pte) & H_PAGE_COMBO) {
|
||||
/*
|
||||
* Make sure we order the hidx load against the _PAGE_COMBO
|
||||
* Make sure we order the hidx load against the H_PAGE_COMBO
|
||||
* check. The store side ordering is done in __hash_page_4K
|
||||
*/
|
||||
smp_rmb();
|
||||
@@ -97,9 +68,9 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
|
||||
|
||||
static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
|
||||
{
|
||||
if ((pte_val(rpte.pte) & _PAGE_COMBO))
|
||||
if ((pte_val(rpte.pte) & H_PAGE_COMBO))
|
||||
return (rpte.hidx >> (index<<2)) & 0xf;
|
||||
return (pte_val(rpte.pte) >> _PAGE_F_GIX_SHIFT) & 0xf;
|
||||
return (pte_val(rpte.pte) >> H_PAGE_F_GIX_SHIFT) & 0xf;
|
||||
}
|
||||
|
||||
#define __rpte_to_pte(r) ((r).pte)
|
||||
@@ -122,79 +93,32 @@ extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index);
|
||||
#define pte_iterate_hashed_end() } while(0); } } while(0)
|
||||
|
||||
#define pte_pagesize_index(mm, addr, pte) \
|
||||
(((pte) & _PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K)
|
||||
(((pte) & H_PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K)
|
||||
|
||||
#define remap_4k_pfn(vma, addr, pfn, prot) \
|
||||
(WARN_ON(((pfn) >= (1UL << PTE_RPN_SIZE))) ? -EINVAL : \
|
||||
remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, \
|
||||
__pgprot(pgprot_val((prot)) | _PAGE_4K_PFN)))
|
||||
extern int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
|
||||
unsigned long pfn, unsigned long size, pgprot_t);
|
||||
static inline int hash__remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,
|
||||
unsigned long pfn, pgprot_t prot)
|
||||
{
|
||||
if (pfn > (PTE_RPN_MASK >> PAGE_SHIFT)) {
|
||||
WARN(1, "remap_4k_pfn called with wrong pfn value\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
return remap_pfn_range(vma, addr, pfn, PAGE_SIZE,
|
||||
__pgprot(pgprot_val(prot) | H_PAGE_4K_PFN));
|
||||
}
|
||||
|
||||
#define PTE_TABLE_SIZE PTE_FRAG_SIZE
|
||||
#define H_PTE_TABLE_SIZE PTE_FRAG_SIZE
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
#define PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + (sizeof(unsigned long) << PMD_INDEX_SIZE))
|
||||
#define H_PMD_TABLE_SIZE ((sizeof(pmd_t) << PMD_INDEX_SIZE) + \
|
||||
(sizeof(unsigned long) << PMD_INDEX_SIZE))
|
||||
#else
|
||||
#define PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
|
||||
#define H_PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE)
|
||||
#endif
|
||||
#define PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
|
||||
#define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
|
||||
|
||||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
/*
|
||||
* We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have
|
||||
* 16GB hugepage pte in PGD and 16MB hugepage pte at PMD;
|
||||
*
|
||||
* Defined in such a way that we can optimize away code block at build time
|
||||
* if CONFIG_HUGETLB_PAGE=n.
|
||||
*/
|
||||
static inline int pmd_huge(pmd_t pmd)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
return !!(pmd_val(pmd) & _PAGE_PTE);
|
||||
}
|
||||
|
||||
static inline int pud_huge(pud_t pud)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
return !!(pud_val(pud) & _PAGE_PTE);
|
||||
}
|
||||
|
||||
static inline int pgd_huge(pgd_t pgd)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
return !!(pgd_val(pgd) & _PAGE_PTE);
|
||||
}
|
||||
#define pgd_huge pgd_huge
|
||||
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
extern int hugepd_ok(hugepd_t hpd);
|
||||
#define is_hugepd(hpd) (hugepd_ok(hpd))
|
||||
#else
|
||||
/*
|
||||
* With 64k page size, we have hugepage ptes in the pgd and pmd entries. We don't
|
||||
* need to setup hugepage directory for them. Our pte and page directory format
|
||||
* enable us to have this enabled.
|
||||
*/
|
||||
static inline int hugepd_ok(hugepd_t hpd)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#define is_hugepd(pdep) 0
|
||||
#endif /* CONFIG_DEBUG_VM */
|
||||
|
||||
#endif /* CONFIG_HUGETLB_PAGE */
|
||||
#define H_PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE)
|
||||
#define H_PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE)
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
extern unsigned long pmd_hugepage_update(struct mm_struct *mm,
|
||||
unsigned long addr,
|
||||
pmd_t *pmdp,
|
||||
unsigned long clr,
|
||||
unsigned long set);
|
||||
static inline char *get_hpte_slot_array(pmd_t *pmdp)
|
||||
{
|
||||
/*
|
||||
@@ -253,50 +177,35 @@ static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array,
|
||||
* that for explicit huge pages.
|
||||
*
|
||||
*/
|
||||
static inline int pmd_trans_huge(pmd_t pmd)
|
||||
static inline int hash__pmd_trans_huge(pmd_t pmd)
|
||||
{
|
||||
return !!((pmd_val(pmd) & (_PAGE_PTE | _PAGE_THP_HUGE)) ==
|
||||
(_PAGE_PTE | _PAGE_THP_HUGE));
|
||||
return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE)) ==
|
||||
(_PAGE_PTE | H_PAGE_THP_HUGE));
|
||||
}
|
||||
|
||||
static inline int pmd_large(pmd_t pmd)
|
||||
static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b)
|
||||
{
|
||||
return !!(pmd_val(pmd) & _PAGE_PTE);
|
||||
return (((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0);
|
||||
}
|
||||
|
||||
static inline pmd_t pmd_mknotpresent(pmd_t pmd)
|
||||
static inline pmd_t hash__pmd_mkhuge(pmd_t pmd)
|
||||
{
|
||||
return __pmd(pmd_val(pmd) & ~_PAGE_PRESENT);
|
||||
}
|
||||
|
||||
#define __HAVE_ARCH_PMD_SAME
|
||||
static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
|
||||
{
|
||||
return (((pmd_val(pmd_a) ^ pmd_val(pmd_b)) & ~_PAGE_HPTEFLAGS) == 0);
|
||||
}
|
||||
|
||||
static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp)
|
||||
{
|
||||
unsigned long old;
|
||||
|
||||
if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
|
||||
return 0;
|
||||
old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
|
||||
return ((old & _PAGE_ACCESSED) != 0);
|
||||
}
|
||||
|
||||
#define __HAVE_ARCH_PMDP_SET_WRPROTECT
|
||||
static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp)
|
||||
{
|
||||
|
||||
if ((pmd_val(*pmdp) & _PAGE_RW) == 0)
|
||||
return;
|
||||
|
||||
pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW, 0);
|
||||
return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE));
|
||||
}
|
||||
|
||||
extern unsigned long hash__pmd_hugepage_update(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp,
|
||||
unsigned long clr, unsigned long set);
|
||||
extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp);
|
||||
extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pgtable_t pgtable);
|
||||
extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
|
||||
extern void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp);
|
||||
extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp);
|
||||
extern int hash__has_transparent_hugepage(void);
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,14 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_HUGETLB_RADIX_H
|
||||
#define _ASM_POWERPC_BOOK3S_64_HUGETLB_RADIX_H
|
||||
/*
|
||||
* For radix we want generic code to handle hugetlb. But then if we want
|
||||
* both hash and radix to be enabled together we need to workaround the
|
||||
* limitations.
|
||||
*/
|
||||
void radix__flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
|
||||
void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
|
||||
extern unsigned long
|
||||
radix__hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
|
||||
unsigned long len, unsigned long pgoff,
|
||||
unsigned long flags);
|
||||
#endif
|
||||
@@ -1,5 +1,5 @@
|
||||
#ifndef _ASM_POWERPC_MMU_HASH64_H_
|
||||
#define _ASM_POWERPC_MMU_HASH64_H_
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_
|
||||
#define _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_
|
||||
/*
|
||||
* PowerPC64 memory management structures
|
||||
*
|
||||
@@ -78,6 +78,10 @@
|
||||
#define HPTE_V_SECONDARY ASM_CONST(0x0000000000000002)
|
||||
#define HPTE_V_VALID ASM_CONST(0x0000000000000001)
|
||||
|
||||
/*
|
||||
* ISA 3.0 have a different HPTE format.
|
||||
*/
|
||||
#define HPTE_R_3_0_SSIZE_SHIFT 58
|
||||
#define HPTE_R_PP0 ASM_CONST(0x8000000000000000)
|
||||
#define HPTE_R_TS ASM_CONST(0x4000000000000000)
|
||||
#define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000)
|
||||
@@ -115,6 +119,7 @@
|
||||
#define POWER7_TLB_SETS 128 /* # sets in POWER7 TLB */
|
||||
#define POWER8_TLB_SETS 512 /* # sets in POWER8 TLB */
|
||||
#define POWER9_TLB_SETS_HASH 256 /* # sets in POWER9 TLB Hash mode */
|
||||
#define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
@@ -127,24 +132,6 @@ extern struct hash_pte *htab_address;
|
||||
extern unsigned long htab_size_bytes;
|
||||
extern unsigned long htab_hash_mask;
|
||||
|
||||
/*
|
||||
* Page size definition
|
||||
*
|
||||
* shift : is the "PAGE_SHIFT" value for that page size
|
||||
* sllp : is a bit mask with the value of SLB L || LP to be or'ed
|
||||
* directly to a slbmte "vsid" value
|
||||
* penc : is the HPTE encoding mask for the "LP" field:
|
||||
*
|
||||
*/
|
||||
struct mmu_psize_def
|
||||
{
|
||||
unsigned int shift; /* number of bits */
|
||||
int penc[MMU_PAGE_COUNT]; /* HPTE encoding */
|
||||
unsigned int tlbiel; /* tlbiel supported for that page size */
|
||||
unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */
|
||||
unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */
|
||||
};
|
||||
extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
|
||||
|
||||
static inline int shift_to_mmu_psize(unsigned int shift)
|
||||
{
|
||||
@@ -210,11 +197,6 @@ static inline int segment_shift(int ssize)
|
||||
/*
|
||||
* The current system page and segment sizes
|
||||
*/
|
||||
extern int mmu_linear_psize;
|
||||
extern int mmu_virtual_psize;
|
||||
extern int mmu_vmalloc_psize;
|
||||
extern int mmu_vmemmap_psize;
|
||||
extern int mmu_io_psize;
|
||||
extern int mmu_kernel_ssize;
|
||||
extern int mmu_highuser_ssize;
|
||||
extern u16 mmu_slb_size;
|
||||
@@ -247,7 +229,8 @@ static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize,
|
||||
*/
|
||||
v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm);
|
||||
v <<= HPTE_V_AVPN_SHIFT;
|
||||
v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
|
||||
if (!cpu_has_feature(CPU_FTR_ARCH_300))
|
||||
v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
|
||||
return v;
|
||||
}
|
||||
|
||||
@@ -271,8 +254,12 @@ static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize,
|
||||
* aligned for the requested page size
|
||||
*/
|
||||
static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize,
|
||||
int actual_psize)
|
||||
int actual_psize, int ssize)
|
||||
{
|
||||
|
||||
if (cpu_has_feature(CPU_FTR_ARCH_300))
|
||||
pa |= ((unsigned long) ssize) << HPTE_R_3_0_SSIZE_SHIFT;
|
||||
|
||||
/* A 4K page needs no special encoding */
|
||||
if (actual_psize == MMU_PAGE_4K)
|
||||
return pa & HPTE_R_RPN;
|
||||
@@ -476,7 +463,7 @@ extern void slb_set_size(u16 size);
|
||||
add rt,rt,rx
|
||||
|
||||
/* 4 bits per slice and we have one slice per 1TB */
|
||||
#define SLICE_ARRAY_SIZE (PGTABLE_RANGE >> 41)
|
||||
#define SLICE_ARRAY_SIZE (H_PGTABLE_RANGE >> 41)
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
@@ -512,38 +499,6 @@ static inline void subpage_prot_free(struct mm_struct *mm) {}
|
||||
static inline void subpage_prot_init_new_context(struct mm_struct *mm) { }
|
||||
#endif /* CONFIG_PPC_SUBPAGE_PROT */
|
||||
|
||||
typedef unsigned long mm_context_id_t;
|
||||
struct spinlock;
|
||||
|
||||
typedef struct {
|
||||
mm_context_id_t id;
|
||||
u16 user_psize; /* page size index */
|
||||
|
||||
#ifdef CONFIG_PPC_MM_SLICES
|
||||
u64 low_slices_psize; /* SLB page size encodings */
|
||||
unsigned char high_slices_psize[SLICE_ARRAY_SIZE];
|
||||
#else
|
||||
u16 sllp; /* SLB page size encoding */
|
||||
#endif
|
||||
unsigned long vdso_base;
|
||||
#ifdef CONFIG_PPC_SUBPAGE_PROT
|
||||
struct subpage_prot_table spt;
|
||||
#endif /* CONFIG_PPC_SUBPAGE_PROT */
|
||||
#ifdef CONFIG_PPC_ICSWX
|
||||
struct spinlock *cop_lockp; /* guard acop and cop_pid */
|
||||
unsigned long acop; /* mask of enabled coprocessor types */
|
||||
unsigned int cop_pid; /* pid value used with coprocessors */
|
||||
#endif /* CONFIG_PPC_ICSWX */
|
||||
#ifdef CONFIG_PPC_64K_PAGES
|
||||
/* for 4K PTE fragment support */
|
||||
void *pte_frag;
|
||||
#endif
|
||||
#ifdef CONFIG_SPAPR_TCE_IOMMU
|
||||
struct list_head iommu_group_mem_list;
|
||||
#endif
|
||||
} mm_context_t;
|
||||
|
||||
|
||||
#if 0
|
||||
/*
|
||||
* The code below is equivalent to this function for arguments
|
||||
@@ -579,7 +534,7 @@ static inline unsigned long get_vsid(unsigned long context, unsigned long ea,
|
||||
/*
|
||||
* Bad address. We return VSID 0 for that
|
||||
*/
|
||||
if ((ea & ~REGION_MASK) >= PGTABLE_RANGE)
|
||||
if ((ea & ~REGION_MASK) >= H_PGTABLE_RANGE)
|
||||
return 0;
|
||||
|
||||
if (ssize == MMU_SEGSIZE_256M)
|
||||
@@ -613,4 +568,4 @@ unsigned htab_shift_for_mem_size(unsigned long mem_size);
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#endif /* _ASM_POWERPC_MMU_HASH64_H_ */
|
||||
#endif /* _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_ */
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_MMU_H_
|
||||
#define _ASM_POWERPC_BOOK3S_64_MMU_H_
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
/*
|
||||
* Page size definition
|
||||
*
|
||||
* shift : is the "PAGE_SHIFT" value for that page size
|
||||
* sllp : is a bit mask with the value of SLB L || LP to be or'ed
|
||||
* directly to a slbmte "vsid" value
|
||||
* penc : is the HPTE encoding mask for the "LP" field:
|
||||
*
|
||||
*/
|
||||
struct mmu_psize_def {
|
||||
unsigned int shift; /* number of bits */
|
||||
int penc[MMU_PAGE_COUNT]; /* HPTE encoding */
|
||||
unsigned int tlbiel; /* tlbiel supported for that page size */
|
||||
unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */
|
||||
union {
|
||||
unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */
|
||||
unsigned long ap; /* Ap encoding used by PowerISA 3.0 */
|
||||
};
|
||||
};
|
||||
extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
|
||||
|
||||
#define radix_enabled() mmu_has_feature(MMU_FTR_RADIX)
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
/* 64-bit classic hash table MMU */
|
||||
#include <asm/book3s/64/mmu-hash.h>
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
/*
|
||||
* ISA 3.0 partiton and process table entry format
|
||||
*/
|
||||
struct prtb_entry {
|
||||
__be64 prtb0;
|
||||
__be64 prtb1;
|
||||
};
|
||||
extern struct prtb_entry *process_tb;
|
||||
|
||||
struct patb_entry {
|
||||
__be64 patb0;
|
||||
__be64 patb1;
|
||||
};
|
||||
extern struct patb_entry *partition_tb;
|
||||
|
||||
#define PATB_HR (1UL << 63)
|
||||
#define PATB_GR (1UL << 63)
|
||||
#define RPDB_MASK 0x0ffffffffffff00fUL
|
||||
#define RPDB_SHIFT (1UL << 8)
|
||||
/*
|
||||
* Limit process table to PAGE_SIZE table. This
|
||||
* also limit the max pid we can support.
|
||||
* MAX_USER_CONTEXT * 16 bytes of space.
|
||||
*/
|
||||
#define PRTB_SIZE_SHIFT (CONTEXT_BITS + 4)
|
||||
/*
|
||||
* Power9 currently only support 64K partition table size.
|
||||
*/
|
||||
#define PATB_SIZE_SHIFT 16
|
||||
|
||||
typedef unsigned long mm_context_id_t;
|
||||
struct spinlock;
|
||||
|
||||
typedef struct {
|
||||
mm_context_id_t id;
|
||||
u16 user_psize; /* page size index */
|
||||
|
||||
#ifdef CONFIG_PPC_MM_SLICES
|
||||
u64 low_slices_psize; /* SLB page size encodings */
|
||||
unsigned char high_slices_psize[SLICE_ARRAY_SIZE];
|
||||
#else
|
||||
u16 sllp; /* SLB page size encoding */
|
||||
#endif
|
||||
unsigned long vdso_base;
|
||||
#ifdef CONFIG_PPC_SUBPAGE_PROT
|
||||
struct subpage_prot_table spt;
|
||||
#endif /* CONFIG_PPC_SUBPAGE_PROT */
|
||||
#ifdef CONFIG_PPC_ICSWX
|
||||
struct spinlock *cop_lockp; /* guard acop and cop_pid */
|
||||
unsigned long acop; /* mask of enabled coprocessor types */
|
||||
unsigned int cop_pid; /* pid value used with coprocessors */
|
||||
#endif /* CONFIG_PPC_ICSWX */
|
||||
#ifdef CONFIG_PPC_64K_PAGES
|
||||
/* for 4K PTE fragment support */
|
||||
void *pte_frag;
|
||||
#endif
|
||||
#ifdef CONFIG_SPAPR_TCE_IOMMU
|
||||
struct list_head iommu_group_mem_list;
|
||||
#endif
|
||||
} mm_context_t;
|
||||
|
||||
/*
|
||||
* The current system page and segment sizes
|
||||
*/
|
||||
extern int mmu_linear_psize;
|
||||
extern int mmu_virtual_psize;
|
||||
extern int mmu_vmalloc_psize;
|
||||
extern int mmu_vmemmap_psize;
|
||||
extern int mmu_io_psize;
|
||||
|
||||
/* MMU initialization */
|
||||
extern void radix_init_native(void);
|
||||
extern void hash__early_init_mmu(void);
|
||||
extern void radix__early_init_mmu(void);
|
||||
static inline void early_init_mmu(void)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__early_init_mmu();
|
||||
return hash__early_init_mmu();
|
||||
}
|
||||
extern void hash__early_init_mmu_secondary(void);
|
||||
extern void radix__early_init_mmu_secondary(void);
|
||||
static inline void early_init_mmu_secondary(void)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__early_init_mmu_secondary();
|
||||
return hash__early_init_mmu_secondary();
|
||||
}
|
||||
|
||||
extern void hash__setup_initial_memory_limit(phys_addr_t first_memblock_base,
|
||||
phys_addr_t first_memblock_size);
|
||||
extern void radix__setup_initial_memory_limit(phys_addr_t first_memblock_base,
|
||||
phys_addr_t first_memblock_size);
|
||||
static inline void setup_initial_memory_limit(phys_addr_t first_memblock_base,
|
||||
phys_addr_t first_memblock_size)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__setup_initial_memory_limit(first_memblock_base,
|
||||
first_memblock_size);
|
||||
return hash__setup_initial_memory_limit(first_memblock_base,
|
||||
first_memblock_size);
|
||||
}
|
||||
#endif /* __ASSEMBLY__ */
|
||||
#endif /* _ASM_POWERPC_BOOK3S_64_MMU_H_ */
|
||||
@@ -0,0 +1,207 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_PGALLOC_H
|
||||
#define _ASM_POWERPC_BOOK3S_64_PGALLOC_H
|
||||
/*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version
|
||||
* 2 of the License, or (at your option) any later version.
|
||||
*/
|
||||
|
||||
#include <linux/slab.h>
|
||||
#include <linux/cpumask.h>
|
||||
#include <linux/percpu.h>
|
||||
|
||||
struct vmemmap_backing {
|
||||
struct vmemmap_backing *list;
|
||||
unsigned long phys;
|
||||
unsigned long virt_addr;
|
||||
};
|
||||
extern struct vmemmap_backing *vmemmap_list;
|
||||
|
||||
/*
|
||||
* Functions that deal with pagetables that could be at any level of
|
||||
* the table need to be passed an "index_size" so they know how to
|
||||
* handle allocation. For PTE pages (which are linked to a struct
|
||||
* page for now, and drawn from the main get_free_pages() pool), the
|
||||
* allocation size will be (2^index_size * sizeof(pointer)) and
|
||||
* allocations are drawn from the kmem_cache in PGT_CACHE(index_size).
|
||||
*
|
||||
* The maximum index size needs to be big enough to allow any
|
||||
* pagetable sizes we need, but small enough to fit in the low bits of
|
||||
* any page table pointer. In other words all pagetables, even tiny
|
||||
* ones, must be aligned to allow at least enough low 0 bits to
|
||||
* contain this value. This value is also used as a mask, so it must
|
||||
* be one less than a power of two.
|
||||
*/
|
||||
#define MAX_PGTABLE_INDEX_SIZE 0xf
|
||||
|
||||
extern struct kmem_cache *pgtable_cache[];
|
||||
#define PGT_CACHE(shift) ({ \
|
||||
BUG_ON(!(shift)); \
|
||||
pgtable_cache[(shift) - 1]; \
|
||||
})
|
||||
|
||||
#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO
|
||||
|
||||
extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int);
|
||||
extern void pte_fragment_free(unsigned long *, int);
|
||||
extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
|
||||
#ifdef CONFIG_SMP
|
||||
extern void __tlb_remove_table(void *_table);
|
||||
#endif
|
||||
|
||||
static inline pgd_t *radix__pgd_alloc(struct mm_struct *mm)
|
||||
{
|
||||
#ifdef CONFIG_PPC_64K_PAGES
|
||||
return (pgd_t *)__get_free_page(PGALLOC_GFP);
|
||||
#else
|
||||
struct page *page;
|
||||
page = alloc_pages(PGALLOC_GFP, 4);
|
||||
if (!page)
|
||||
return NULL;
|
||||
return (pgd_t *) page_address(page);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd)
|
||||
{
|
||||
#ifdef CONFIG_PPC_64K_PAGES
|
||||
free_page((unsigned long)pgd);
|
||||
#else
|
||||
free_pages((unsigned long)pgd, 4);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__pgd_alloc(mm);
|
||||
return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
|
||||
}
|
||||
|
||||
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__pgd_free(mm, pgd);
|
||||
kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
|
||||
}
|
||||
|
||||
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
|
||||
{
|
||||
pgd_set(pgd, __pgtable_ptr_val(pud) | PGD_VAL_BITS);
|
||||
}
|
||||
|
||||
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
|
||||
{
|
||||
return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
|
||||
GFP_KERNEL|__GFP_REPEAT);
|
||||
}
|
||||
|
||||
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
|
||||
{
|
||||
kmem_cache_free(PGT_CACHE(PUD_INDEX_SIZE), pud);
|
||||
}
|
||||
|
||||
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
|
||||
{
|
||||
pud_set(pud, __pgtable_ptr_val(pmd) | PUD_VAL_BITS);
|
||||
}
|
||||
|
||||
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
|
||||
unsigned long address)
|
||||
{
|
||||
pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE);
|
||||
}
|
||||
|
||||
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
|
||||
{
|
||||
return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
|
||||
GFP_KERNEL|__GFP_REPEAT);
|
||||
}
|
||||
|
||||
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
|
||||
{
|
||||
kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
|
||||
}
|
||||
|
||||
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
|
||||
unsigned long address)
|
||||
{
|
||||
return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);
|
||||
}
|
||||
|
||||
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
|
||||
pte_t *pte)
|
||||
{
|
||||
pmd_set(pmd, __pgtable_ptr_val(pte) | PMD_VAL_BITS);
|
||||
}
|
||||
|
||||
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
|
||||
pgtable_t pte_page)
|
||||
{
|
||||
pmd_set(pmd, __pgtable_ptr_val(pte_page) | PMD_VAL_BITS);
|
||||
}
|
||||
|
||||
static inline pgtable_t pmd_pgtable(pmd_t pmd)
|
||||
{
|
||||
return (pgtable_t)pmd_page_vaddr(pmd);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PPC_4K_PAGES
|
||||
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
|
||||
unsigned long address)
|
||||
{
|
||||
return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
|
||||
}
|
||||
|
||||
static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
|
||||
unsigned long address)
|
||||
{
|
||||
struct page *page;
|
||||
pte_t *pte;
|
||||
|
||||
pte = pte_alloc_one_kernel(mm, address);
|
||||
if (!pte)
|
||||
return NULL;
|
||||
page = virt_to_page(pte);
|
||||
if (!pgtable_page_ctor(page)) {
|
||||
__free_page(page);
|
||||
return NULL;
|
||||
}
|
||||
return pte;
|
||||
}
|
||||
#else /* if CONFIG_PPC_64K_PAGES */
|
||||
|
||||
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
|
||||
unsigned long address)
|
||||
{
|
||||
return (pte_t *)pte_fragment_alloc(mm, address, 1);
|
||||
}
|
||||
|
||||
static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
|
||||
unsigned long address)
|
||||
{
|
||||
return (pgtable_t)pte_fragment_alloc(mm, address, 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
|
||||
{
|
||||
pte_fragment_free((unsigned long *)pte, 1);
|
||||
}
|
||||
|
||||
static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
|
||||
{
|
||||
pte_fragment_free((unsigned long *)ptepage, 0);
|
||||
}
|
||||
|
||||
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
|
||||
unsigned long address)
|
||||
{
|
||||
tlb_flush_pgtable(tlb, address);
|
||||
pgtable_free_tlb(tlb, table, 0);
|
||||
}
|
||||
|
||||
#define check_pgt_cache() do { } while (0)
|
||||
|
||||
#endif /* _ASM_POWERPC_BOOK3S_64_PGALLOC_H */
|
||||
@@ -0,0 +1,53 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H
|
||||
#define _ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H
|
||||
/*
|
||||
* hash 4k can't share hugetlb and also doesn't support THP
|
||||
*/
|
||||
#ifndef __ASSEMBLY__
|
||||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
static inline int pmd_huge(pmd_t pmd)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
if (radix_enabled())
|
||||
return !!(pmd_val(pmd) & _PAGE_PTE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int pud_huge(pud_t pud)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
if (radix_enabled())
|
||||
return !!(pud_val(pud) & _PAGE_PTE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int pgd_huge(pgd_t pgd)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
if (radix_enabled())
|
||||
return !!(pgd_val(pgd) & _PAGE_PTE);
|
||||
return 0;
|
||||
}
|
||||
#define pgd_huge pgd_huge
|
||||
/*
|
||||
* With radix , we have hugepage ptes in the pud and pmd entries. We don't
|
||||
* need to setup hugepage directory for them. Our pte and page directory format
|
||||
* enable us to have this enabled.
|
||||
*/
|
||||
static inline int hugepd_ok(hugepd_t hpd)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return 0;
|
||||
return hash__hugepd_ok(hpd);
|
||||
}
|
||||
#define is_hugepd(hpd) (hugepd_ok(hpd))
|
||||
#endif /* CONFIG_HUGETLB_PAGE */
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#endif /*_ASM_POWERPC_BOOK3S_64_PGTABLE_4K_H */
|
||||
@@ -0,0 +1,64 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H
|
||||
#define _ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
/*
|
||||
* We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have
|
||||
* 16GB hugepage pte in PGD and 16MB hugepage pte at PMD;
|
||||
*
|
||||
* Defined in such a way that we can optimize away code block at build time
|
||||
* if CONFIG_HUGETLB_PAGE=n.
|
||||
*/
|
||||
static inline int pmd_huge(pmd_t pmd)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
return !!(pmd_val(pmd) & _PAGE_PTE);
|
||||
}
|
||||
|
||||
static inline int pud_huge(pud_t pud)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
return !!(pud_val(pud) & _PAGE_PTE);
|
||||
}
|
||||
|
||||
static inline int pgd_huge(pgd_t pgd)
|
||||
{
|
||||
/*
|
||||
* leaf pte for huge page
|
||||
*/
|
||||
return !!(pgd_val(pgd) & _PAGE_PTE);
|
||||
}
|
||||
#define pgd_huge pgd_huge
|
||||
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
extern int hugepd_ok(hugepd_t hpd);
|
||||
#define is_hugepd(hpd) (hugepd_ok(hpd))
|
||||
#else
|
||||
/*
|
||||
* With 64k page size, we have hugepage ptes in the pgd and pmd entries. We don't
|
||||
* need to setup hugepage directory for them. Our pte and page directory format
|
||||
* enable us to have this enabled.
|
||||
*/
|
||||
static inline int hugepd_ok(hugepd_t hpd)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#define is_hugepd(pdep) 0
|
||||
#endif /* CONFIG_DEBUG_VM */
|
||||
|
||||
#endif /* CONFIG_HUGETLB_PAGE */
|
||||
|
||||
static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,
|
||||
unsigned long pfn, pgprot_t prot)
|
||||
{
|
||||
if (radix_enabled())
|
||||
BUG();
|
||||
return hash__remap_4k_pfn(vma, addr, pfn, prot);
|
||||
}
|
||||
#endif /* __ASSEMBLY__ */
|
||||
#endif /*_ASM_POWERPC_BOOK3S_64_PGTABLE_64K_H */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,12 @@
|
||||
#ifndef _ASM_POWERPC_PGTABLE_RADIX_4K_H
|
||||
#define _ASM_POWERPC_PGTABLE_RADIX_4K_H
|
||||
|
||||
/*
|
||||
* For 4K page size supported index is 13/9/9/9
|
||||
*/
|
||||
#define RADIX_PTE_INDEX_SIZE 9 /* 2MB huge page */
|
||||
#define RADIX_PMD_INDEX_SIZE 9 /* 1G huge page */
|
||||
#define RADIX_PUD_INDEX_SIZE 9
|
||||
#define RADIX_PGD_INDEX_SIZE 13
|
||||
|
||||
#endif /* _ASM_POWERPC_PGTABLE_RADIX_4K_H */
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef _ASM_POWERPC_PGTABLE_RADIX_64K_H
|
||||
#define _ASM_POWERPC_PGTABLE_RADIX_64K_H
|
||||
|
||||
/*
|
||||
* For 64K page size supported index is 13/9/9/5
|
||||
*/
|
||||
#define RADIX_PTE_INDEX_SIZE 5 /* 2MB huge page */
|
||||
#define RADIX_PMD_INDEX_SIZE 9 /* 1G huge page */
|
||||
#define RADIX_PUD_INDEX_SIZE 9
|
||||
#define RADIX_PGD_INDEX_SIZE 13
|
||||
|
||||
#endif /* _ASM_POWERPC_PGTABLE_RADIX_64K_H */
|
||||
@@ -0,0 +1,232 @@
|
||||
#ifndef _ASM_POWERPC_PGTABLE_RADIX_H
|
||||
#define _ASM_POWERPC_PGTABLE_RADIX_H
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
#include <asm/cmpxchg.h>
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_PPC_64K_PAGES
|
||||
#include <asm/book3s/64/radix-64k.h>
|
||||
#else
|
||||
#include <asm/book3s/64/radix-4k.h>
|
||||
#endif
|
||||
|
||||
/* An empty PTE can still have a R or C writeback */
|
||||
#define RADIX_PTE_NONE_MASK (_PAGE_DIRTY | _PAGE_ACCESSED)
|
||||
|
||||
/* Bits to set in a RPMD/RPUD/RPGD */
|
||||
#define RADIX_PMD_VAL_BITS (0x8000000000000000UL | RADIX_PTE_INDEX_SIZE)
|
||||
#define RADIX_PUD_VAL_BITS (0x8000000000000000UL | RADIX_PMD_INDEX_SIZE)
|
||||
#define RADIX_PGD_VAL_BITS (0x8000000000000000UL | RADIX_PUD_INDEX_SIZE)
|
||||
|
||||
/* Don't have anything in the reserved bits and leaf bits */
|
||||
#define RADIX_PMD_BAD_BITS 0x60000000000000e0UL
|
||||
#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL
|
||||
#define RADIX_PGD_BAD_BITS 0x60000000000000e0UL
|
||||
|
||||
/*
|
||||
* Size of EA range mapped by our pagetables.
|
||||
*/
|
||||
#define RADIX_PGTABLE_EADDR_SIZE (RADIX_PTE_INDEX_SIZE + RADIX_PMD_INDEX_SIZE + \
|
||||
RADIX_PUD_INDEX_SIZE + RADIX_PGD_INDEX_SIZE + PAGE_SHIFT)
|
||||
#define RADIX_PGTABLE_RANGE (ASM_CONST(1) << RADIX_PGTABLE_EADDR_SIZE)
|
||||
|
||||
/*
|
||||
* We support 52 bit address space, Use top bit for kernel
|
||||
* virtual mapping. Also make sure kernel fit in the top
|
||||
* quadrant.
|
||||
*
|
||||
* +------------------+
|
||||
* +------------------+ Kernel virtual map (0xc008000000000000)
|
||||
* | |
|
||||
* | |
|
||||
* | |
|
||||
* 0b11......+------------------+ Kernel linear map (0xc....)
|
||||
* | |
|
||||
* | 2 quadrant |
|
||||
* | |
|
||||
* 0b10......+------------------+
|
||||
* | |
|
||||
* | 1 quadrant |
|
||||
* | |
|
||||
* 0b01......+------------------+
|
||||
* | |
|
||||
* | 0 quadrant |
|
||||
* | |
|
||||
* 0b00......+------------------+
|
||||
*
|
||||
*
|
||||
* 3rd quadrant expanded:
|
||||
* +------------------------------+
|
||||
* | |
|
||||
* | |
|
||||
* | |
|
||||
* +------------------------------+ Kernel IO map end (0xc010000000000000)
|
||||
* | |
|
||||
* | |
|
||||
* | 1/2 of virtual map |
|
||||
* | |
|
||||
* | |
|
||||
* +------------------------------+ Kernel IO map start
|
||||
* | |
|
||||
* | 1/4 of virtual map |
|
||||
* | |
|
||||
* +------------------------------+ Kernel vmemap start
|
||||
* | |
|
||||
* | 1/4 of virtual map |
|
||||
* | |
|
||||
* +------------------------------+ Kernel virt start (0xc008000000000000)
|
||||
* | |
|
||||
* | |
|
||||
* | |
|
||||
* +------------------------------+ Kernel linear (0xc.....)
|
||||
*/
|
||||
|
||||
#define RADIX_KERN_VIRT_START ASM_CONST(0xc008000000000000)
|
||||
#define RADIX_KERN_VIRT_SIZE ASM_CONST(0x0008000000000000)
|
||||
|
||||
/*
|
||||
* The vmalloc space starts at the beginning of that region, and
|
||||
* occupies a quarter of it on radix config.
|
||||
* (we keep a quarter for the virtual memmap)
|
||||
*/
|
||||
#define RADIX_VMALLOC_START RADIX_KERN_VIRT_START
|
||||
#define RADIX_VMALLOC_SIZE (RADIX_KERN_VIRT_SIZE >> 2)
|
||||
#define RADIX_VMALLOC_END (RADIX_VMALLOC_START + RADIX_VMALLOC_SIZE)
|
||||
/*
|
||||
* Defines the address of the vmemap area, in its own region on
|
||||
* hash table CPUs.
|
||||
*/
|
||||
#define RADIX_VMEMMAP_BASE (RADIX_VMALLOC_END)
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
#define RADIX_PTE_TABLE_SIZE (sizeof(pte_t) << RADIX_PTE_INDEX_SIZE)
|
||||
#define RADIX_PMD_TABLE_SIZE (sizeof(pmd_t) << RADIX_PMD_INDEX_SIZE)
|
||||
#define RADIX_PUD_TABLE_SIZE (sizeof(pud_t) << RADIX_PUD_INDEX_SIZE)
|
||||
#define RADIX_PGD_TABLE_SIZE (sizeof(pgd_t) << RADIX_PGD_INDEX_SIZE)
|
||||
|
||||
static inline unsigned long radix__pte_update(struct mm_struct *mm,
|
||||
unsigned long addr,
|
||||
pte_t *ptep, unsigned long clr,
|
||||
unsigned long set,
|
||||
int huge)
|
||||
{
|
||||
pte_t pte;
|
||||
unsigned long old_pte, new_pte;
|
||||
|
||||
do {
|
||||
pte = READ_ONCE(*ptep);
|
||||
old_pte = pte_val(pte);
|
||||
new_pte = (old_pte | set) & ~clr;
|
||||
|
||||
} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
|
||||
|
||||
/* We already do a sync in cmpxchg, is ptesync needed ?*/
|
||||
asm volatile("ptesync" : : : "memory");
|
||||
/* huge pages use the old page table lock */
|
||||
if (!huge)
|
||||
assert_pte_locked(mm, addr);
|
||||
|
||||
return old_pte;
|
||||
}
|
||||
|
||||
/*
|
||||
* Set the dirty and/or accessed bits atomically in a linux PTE, this
|
||||
* function doesn't need to invalidate tlb.
|
||||
*/
|
||||
static inline void radix__ptep_set_access_flags(pte_t *ptep, pte_t entry)
|
||||
{
|
||||
pte_t pte;
|
||||
unsigned long old_pte, new_pte;
|
||||
unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
|
||||
_PAGE_RW | _PAGE_EXEC);
|
||||
do {
|
||||
pte = READ_ONCE(*ptep);
|
||||
old_pte = pte_val(pte);
|
||||
new_pte = old_pte | set;
|
||||
|
||||
} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
|
||||
|
||||
/* We already do a sync in cmpxchg, is ptesync needed ?*/
|
||||
asm volatile("ptesync" : : : "memory");
|
||||
}
|
||||
|
||||
static inline int radix__pte_same(pte_t pte_a, pte_t pte_b)
|
||||
{
|
||||
return ((pte_raw(pte_a) ^ pte_raw(pte_b)) == 0);
|
||||
}
|
||||
|
||||
static inline int radix__pte_none(pte_t pte)
|
||||
{
|
||||
return (pte_val(pte) & ~RADIX_PTE_NONE_MASK) == 0;
|
||||
}
|
||||
|
||||
static inline void radix__set_pte_at(struct mm_struct *mm, unsigned long addr,
|
||||
pte_t *ptep, pte_t pte, int percpu)
|
||||
{
|
||||
*ptep = pte;
|
||||
asm volatile("ptesync" : : : "memory");
|
||||
}
|
||||
|
||||
static inline int radix__pmd_bad(pmd_t pmd)
|
||||
{
|
||||
return !!(pmd_val(pmd) & RADIX_PMD_BAD_BITS);
|
||||
}
|
||||
|
||||
static inline int radix__pmd_same(pmd_t pmd_a, pmd_t pmd_b)
|
||||
{
|
||||
return ((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) == 0);
|
||||
}
|
||||
|
||||
static inline int radix__pud_bad(pud_t pud)
|
||||
{
|
||||
return !!(pud_val(pud) & RADIX_PUD_BAD_BITS);
|
||||
}
|
||||
|
||||
|
||||
static inline int radix__pgd_bad(pgd_t pgd)
|
||||
{
|
||||
return !!(pgd_val(pgd) & RADIX_PGD_BAD_BITS);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
|
||||
static inline int radix__pmd_trans_huge(pmd_t pmd)
|
||||
{
|
||||
return !!(pmd_val(pmd) & _PAGE_PTE);
|
||||
}
|
||||
|
||||
static inline pmd_t radix__pmd_mkhuge(pmd_t pmd)
|
||||
{
|
||||
return __pmd(pmd_val(pmd) | _PAGE_PTE);
|
||||
}
|
||||
static inline void radix__pmdp_huge_split_prepare(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp)
|
||||
{
|
||||
/* Nothing to do for radix. */
|
||||
return;
|
||||
}
|
||||
|
||||
extern unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, unsigned long clr,
|
||||
unsigned long set);
|
||||
extern pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp);
|
||||
extern void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pgtable_t pgtable);
|
||||
extern pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
|
||||
extern pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp);
|
||||
extern int radix__has_transparent_hugepage(void);
|
||||
#endif
|
||||
|
||||
extern int __meminit radix__vmemmap_create_mapping(unsigned long start,
|
||||
unsigned long page_size,
|
||||
unsigned long phys);
|
||||
extern void radix__vmemmap_remove_mapping(unsigned long start,
|
||||
unsigned long page_size);
|
||||
|
||||
extern int radix__map_kernel_page(unsigned long ea, unsigned long pa,
|
||||
pgprot_t flags, unsigned int psz);
|
||||
#endif /* __ASSEMBLY__ */
|
||||
#endif
|
||||
@@ -1,8 +1,6 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H
|
||||
#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H
|
||||
|
||||
#define MMU_NO_CONTEXT 0
|
||||
|
||||
/*
|
||||
* TLB flushing for 64-bit hash-MMU CPUs
|
||||
*/
|
||||
@@ -29,14 +27,21 @@ extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch);
|
||||
|
||||
static inline void arch_enter_lazy_mmu_mode(void)
|
||||
{
|
||||
struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
|
||||
struct ppc64_tlb_batch *batch;
|
||||
|
||||
if (radix_enabled())
|
||||
return;
|
||||
batch = this_cpu_ptr(&ppc64_tlb_batch);
|
||||
batch->active = 1;
|
||||
}
|
||||
|
||||
static inline void arch_leave_lazy_mmu_mode(void)
|
||||
{
|
||||
struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
|
||||
struct ppc64_tlb_batch *batch;
|
||||
|
||||
if (radix_enabled())
|
||||
return;
|
||||
batch = this_cpu_ptr(&ppc64_tlb_batch);
|
||||
|
||||
if (batch->index)
|
||||
__flush_tlb_pending(batch);
|
||||
@@ -52,40 +57,42 @@ extern void flush_hash_range(unsigned long number, int local);
|
||||
extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
|
||||
pmd_t *pmdp, unsigned int psize, int ssize,
|
||||
unsigned long flags);
|
||||
|
||||
static inline void local_flush_tlb_mm(struct mm_struct *mm)
|
||||
static inline void hash__local_flush_tlb_mm(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void flush_tlb_mm(struct mm_struct *mm)
|
||||
static inline void hash__flush_tlb_mm(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void local_flush_tlb_page(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
static inline void hash__local_flush_tlb_page(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void flush_tlb_page(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
static inline void hash__flush_tlb_page(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void flush_tlb_page_nohash(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
static inline void hash__flush_tlb_page_nohash(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void flush_tlb_range(struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end)
|
||||
static inline void hash__flush_tlb_range(struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void flush_tlb_kernel_range(unsigned long start,
|
||||
unsigned long end)
|
||||
static inline void hash__flush_tlb_kernel_range(unsigned long start,
|
||||
unsigned long end)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
struct mmu_gather;
|
||||
extern void hash__tlb_flush(struct mmu_gather *tlb);
|
||||
/* Private function for use by PCI IO mapping code */
|
||||
extern void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,
|
||||
unsigned long end);
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
#ifndef _ASM_POWERPC_TLBFLUSH_RADIX_H
|
||||
#define _ASM_POWERPC_TLBFLUSH_RADIX_H
|
||||
|
||||
struct vm_area_struct;
|
||||
struct mm_struct;
|
||||
struct mmu_gather;
|
||||
|
||||
static inline int mmu_get_ap(int psize)
|
||||
{
|
||||
return mmu_psize_defs[psize].ap;
|
||||
}
|
||||
|
||||
extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
|
||||
unsigned long end);
|
||||
extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
|
||||
|
||||
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
|
||||
extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
|
||||
extern void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
|
||||
unsigned long ap, int nid);
|
||||
extern void radix__tlb_flush(struct mmu_gather *tlb);
|
||||
#ifdef CONFIG_SMP
|
||||
extern void radix__flush_tlb_mm(struct mm_struct *mm);
|
||||
extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
|
||||
extern void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
|
||||
unsigned long ap, int nid);
|
||||
#else
|
||||
#define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm)
|
||||
#define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr)
|
||||
#define radix___flush_tlb_page(mm,addr,p,i) radix___local_flush_tlb_page(mm,addr,p,i)
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,76 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
|
||||
#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H
|
||||
|
||||
#define MMU_NO_CONTEXT ~0UL
|
||||
|
||||
|
||||
#include <asm/book3s/64/tlbflush-hash.h>
|
||||
#include <asm/book3s/64/tlbflush-radix.h>
|
||||
|
||||
static inline void flush_tlb_range(struct vm_area_struct *vma,
|
||||
unsigned long start, unsigned long end)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__flush_tlb_range(vma, start, end);
|
||||
return hash__flush_tlb_range(vma, start, end);
|
||||
}
|
||||
|
||||
static inline void flush_tlb_kernel_range(unsigned long start,
|
||||
unsigned long end)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__flush_tlb_kernel_range(start, end);
|
||||
return hash__flush_tlb_kernel_range(start, end);
|
||||
}
|
||||
|
||||
static inline void local_flush_tlb_mm(struct mm_struct *mm)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__local_flush_tlb_mm(mm);
|
||||
return hash__local_flush_tlb_mm(mm);
|
||||
}
|
||||
|
||||
static inline void local_flush_tlb_page(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__local_flush_tlb_page(vma, vmaddr);
|
||||
return hash__local_flush_tlb_page(vma, vmaddr);
|
||||
}
|
||||
|
||||
static inline void flush_tlb_page_nohash(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__flush_tlb_page(vma, vmaddr);
|
||||
return hash__flush_tlb_page_nohash(vma, vmaddr);
|
||||
}
|
||||
|
||||
static inline void tlb_flush(struct mmu_gather *tlb)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__tlb_flush(tlb);
|
||||
return hash__tlb_flush(tlb);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
static inline void flush_tlb_mm(struct mm_struct *mm)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__flush_tlb_mm(mm);
|
||||
return hash__flush_tlb_mm(mm);
|
||||
}
|
||||
|
||||
static inline void flush_tlb_page(struct vm_area_struct *vma,
|
||||
unsigned long vmaddr)
|
||||
{
|
||||
if (radix_enabled())
|
||||
return radix__flush_tlb_page(vma, vmaddr);
|
||||
return hash__flush_tlb_page(vma, vmaddr);
|
||||
}
|
||||
#else
|
||||
#define flush_tlb_mm(mm) local_flush_tlb_mm(mm)
|
||||
#define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr)
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */
|
||||
@@ -0,0 +1,19 @@
|
||||
#ifndef _ASM_POWERPC_BOOK3S_PGALLOC_H
|
||||
#define _ASM_POWERPC_BOOK3S_PGALLOC_H
|
||||
|
||||
#include <linux/mm.h>
|
||||
|
||||
extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
|
||||
static inline void tlb_flush_pgtable(struct mmu_gather *tlb,
|
||||
unsigned long address)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PPC64
|
||||
#include <asm/book3s/64/pgalloc.h>
|
||||
#else
|
||||
#include <asm/book3s/32/pgalloc.h>
|
||||
#endif
|
||||
|
||||
#endif /* _ASM_POWERPC_BOOK3S_PGALLOC_H */
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user