diff --git a/arch/frv/mm/pgalloc.c b/arch/frv/mm/pgalloc.c
index 1a2e5c8d03a9c7e639ab354705a08cb0916bb443..66f616fb4860a779d5cc1eafa68d48d41b4db0e2 100644
--- a/arch/frv/mm/pgalloc.c
+++ b/arch/frv/mm/pgalloc.c
@@ -28,7 +28,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
return pte;
}
-struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *page;
@@ -37,9 +37,11 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
#else
page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0);
#endif
- if (page)
+ if (page) {
clear_highpage(page);
- flush_dcache_page(page);
+ pgtable_page_ctor(page);
+ flush_dcache_page(page);
+ }
return page;
}
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index f80f90c4d58be2b277dd6bb6d602b43678b6dbad..ac3390f81900c64192c70a14e1a10258201c7090 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -107,19 +107,20 @@ __init_refok pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long add
return pte;
}
-struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *ptepage;
#ifdef CONFIG_HIGHPTE
- gfp_t flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT;
+ gfp_t flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT | __GFP_ZERO;
#else
- gfp_t flags = GFP_KERNEL | __GFP_REPEAT;
+ gfp_t flags = GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO;
#endif
ptepage = alloc_pages(flags, 0);
- if (ptepage)
- clear_highpage(ptepage);
+ if (!ptepage)
+ return NULL;
+ pgtable_page_ctor(ptepage);
return ptepage;
}
@@ -131,11 +132,12 @@ void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-void pte_free(struct mm_struct *mm, struct page *ptepage)
+void pte_free(struct mm_struct *mm, pgtable_t ptepage)
{
#ifdef CONFIG_SMP
hash_page_sync();
#endif
+ pgtable_page_dtor(ptepage);
__free_page(ptepage);
}
diff --git a/arch/ppc/mm/pgtable.c b/arch/ppc/mm/pgtable.c
index 409fcaa4994a598736ec082218dacde0709f2d9a..03a79bff1271fee749e842d56ca21df176e2674d 100644
--- a/arch/ppc/mm/pgtable.c
+++ b/arch/ppc/mm/pgtable.c
@@ -95,7 +95,7 @@ __init_refok pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long add
return pte;
}
-struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *ptepage;
@@ -106,8 +106,10 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
#endif
ptepage = alloc_pages(flags, 0);
- if (ptepage)
+ if (ptepage) {
clear_highpage(ptepage);
+ pgtable_page_ctor(ptepage);
+ }
return ptepage;
}
@@ -119,11 +121,12 @@ void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-void pte_free(struct mm_struct *mm, struct page *ptepage)
+void pte_free(struct mm_struct *mm, pgtable_t ptepage)
{
#ifdef CONFIG_SMP
hash_page_sync();
#endif
+ pgtable_page_dtor(ptepage);
__free_page(ptepage);
}
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index e60e0ae134025f4ef0b4bcbab65c48ca32cb6f60..019f518cd5a0c014c2e12db2e96274c5f65066d2 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -78,6 +78,7 @@ unsigned long *page_table_alloc(int noexec)
clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE);
page->index = (addr_t) table;
}
+ pgtable_page_ctor(page);
table = (unsigned long *) page_to_phys(page);
clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE);
return table;
@@ -87,6 +88,7 @@ void page_table_free(unsigned long *table)
{
unsigned long *shadow = get_shadow_pte(table);
+ pgtable_page_dtor(virt_to_page(table));
if (shadow)
free_page((unsigned long) shadow);
free_page((unsigned long) table);
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index dc98e3844a0a94363e1c9c5dc3f74e5522ce43a2..23d3291a3e81e65736c862e03e3b24e3047ffa22 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -489,14 +489,17 @@ srmmu_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
}
-static struct page *
+static pgtable_t
srmmu_pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
unsigned long pte;
+ struct page *page;
if ((pte = (unsigned long)srmmu_pte_alloc_one_kernel(mm, address)) == 0)
return NULL;
- return pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT );
+ page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT );
+ pgtable_page_ctor(page);
+ return page;
}
static void srmmu_free_pte_fast(pte_t *pte)
@@ -504,10 +507,11 @@ static void srmmu_free_pte_fast(pte_t *pte)
srmmu_free_nocache((unsigned long)pte, PTE_SIZE);
}
-static void srmmu_pte_free(struct page *pte)
+static void srmmu_pte_free(pgtable_t pte)
{
unsigned long p;
+ pgtable_page_dtor(pte);
p = (unsigned long)page_address(pte); /* Cached address (for test) */
if (p == 0)
BUG();
diff --git a/arch/sparc/mm/sun4c.c b/arch/sparc/mm/sun4c.c
index 0729305f2f594a83ff6259ec1e5e223ba51ab30c..c0442e8c4b1548a784037d1dbe223e996dcefd95 100644
--- a/arch/sparc/mm/sun4c.c
+++ b/arch/sparc/mm/sun4c.c
@@ -1947,12 +1947,17 @@ static pte_t *sun4c_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long add
return pte;
}
-static struct page *sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address)
+static pgtable_t sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
- pte_t *pte = sun4c_pte_alloc_one_kernel(mm, address);
+ pte_t *pte;
+ struct page *page;
+
+ pte = sun4c_pte_alloc_one_kernel(mm, address);
if (pte == NULL)
return NULL;
- return virt_to_page(pte);
+ page = virt_to_page(pte);
+ pgtable_page_ctor(page);
+ return page;
}
static inline void sun4c_free_pte_fast(pte_t *pte)
@@ -1962,8 +1967,9 @@ static inline void sun4c_free_pte_fast(pte_t *pte)
pgtable_cache_size++;
}
-static void sun4c_pte_free(struct page *pte)
+static void sun4c_pte_free(pgtable_t pte)
{
+ pgtable_page_dtor(pte);
sun4c_free_pte_fast(page_address(pte));
}
diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index d872fdce1d7ec76e9d4b37782b2a9ac9dd108395..2627ce82e918887f2fdf4ad9d9031fd4d6ac26ad 100644
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -354,11 +354,13 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
return pte;
}
-struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *pte;
pte = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
+ if (pte)
+ pgtable_page_ctor(pte);
return pte;
}
diff --git a/arch/x86/mm/pgtable_32.c b/arch/x86/mm/pgtable_32.c
index 6c1914622a88344456f571d2cbf06e58921f451b..73aba7125203764b41a3cb3af48ac6d5d5cee5a4 100644
--- a/arch/x86/mm/pgtable_32.c
+++ b/arch/x86/mm/pgtable_32.c
@@ -183,7 +183,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
return (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
}
-struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *pte;
@@ -192,6 +192,8 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
#else
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
#endif
+ if (pte)
+ pgtable_page_ctor(pte);
return pte;
}
@@ -365,6 +367,7 @@ void check_pgt_cache(void)
void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
+ pgtable_page_dtor(pte);
paravirt_release_pt(page_to_pfn(pte));
tlb_remove_page(tlb, pte);
}
diff --git a/include/asm-alpha/page.h b/include/asm-alpha/page.h
index 05f09f997d829429f0ece5af420105299997afd0..22ff9762d17b63b8379abea371cb816d37dcaae2 100644
--- a/include/asm-alpha/page.h
+++ b/include/asm-alpha/page.h
@@ -62,6 +62,8 @@ typedef unsigned long pgprot_t;
#endif /* STRICT_MM_TYPECHECKS */
+typedef struct page *pgtable_t;
+
#ifdef USE_48_BIT_KSEG
#define PAGE_OFFSET 0xffff800000000000UL
#else
diff --git a/include/asm-alpha/pgalloc.h b/include/asm-alpha/pgalloc.h
index fdbedacc7375158990236e13b546a7fa1c92a8a6..fd090155dccd14367fcde13c425f5d224de7eb21 100644
--- a/include/asm-alpha/pgalloc.h
+++ b/include/asm-alpha/pgalloc.h
@@ -11,10 +11,11 @@
*/
static inline void
-pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
+pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte)
{
pmd_set(pmd, (pte_t *)(page_to_pa(pte) + PAGE_OFFSET));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
static inline void
pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte)
@@ -57,18 +58,23 @@ pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-static inline struct page *
-pte_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline pgtable_t
+pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
- pte_t *pte = pte_alloc_one_kernel(mm, addr);
- if (pte)
- return virt_to_page(pte);
- return NULL;
+ pte_t *pte = pte_alloc_one_kernel(mm, address);
+ struct page *page;
+
+ if (!pte)
+ return NULL;
+ page = virt_to_page(pte);
+ pgtable_page_ctor(page);
+ return page;
}
static inline void
-pte_free(struct mm_struct *mm, struct page *page)
+pte_free(struct mm_struct *mm, pgtable_t page)
{
+ pgtable_page_dtor(page);
__free_page(page);
}
diff --git a/include/asm-arm/page.h b/include/asm-arm/page.h
index 31ff12f4ffb7b5fdb757767a159c4f855c544630..c86f68ee65118b6c7cdd2fec34ee7509abbaae46 100644
--- a/include/asm-arm/page.h
+++ b/include/asm-arm/page.h
@@ -171,6 +171,8 @@ typedef unsigned long pgprot_t;
#endif /* STRICT_MM_TYPECHECKS */
+typedef struct page *pgtable_t;
+
#endif /* CONFIG_MMU */
#include <asm/memory.h>
diff --git a/include/asm-arm/pgalloc.h b/include/asm-arm/pgalloc.h
index fb6c6e3222bddc19131fc59408e12ffbfb824a22..163b0305dd76b3e83176f294eb204d3a389bd3ad 100644
--- a/include/asm-arm/pgalloc.h
+++ b/include/asm-arm/pgalloc.h
@@ -66,7 +66,7 @@ pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr)
return pte;
}
-static inline struct page *
+static inline pgtable_t
pte_alloc_one(struct mm_struct *mm, unsigned long addr)
{
struct page *pte;
@@ -75,6 +75,7 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
if (pte) {
void *page = page_address(pte);
clean_dcache_area(page, sizeof(pte_t) * PTRS_PER_PTE);
+ pgtable_page_ctor(pte);
}
return pte;
@@ -91,8 +92,9 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
}
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
@@ -123,10 +125,11 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
}
static inline void
-pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
+pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
{
__pmd_populate(pmdp, page_to_pfn(ptep) << PAGE_SHIFT | _PAGE_USER_TABLE);
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
#endif /* CONFIG_MMU */
diff --git a/include/asm-avr32/page.h b/include/asm-avr32/page.h
index ee23499cec34df071b4c6a75ae189ed08797f7df..5582968feee853220432ac4b503c038c4beb7d0f 100644
--- a/include/asm-avr32/page.h
+++ b/include/asm-avr32/page.h
@@ -34,6 +34,7 @@ extern void copy_page(void *to, void *from);
typedef struct { unsigned long pte; } pte_t;
typedef struct { unsigned long pgd; } pgd_t;
typedef struct { unsigned long pgprot; } pgprot_t;
+typedef struct page *pgtable_t;
#define pte_val(x) ((x).pte)
#define pgd_val(x) ((x).pgd)
diff --git a/include/asm-avr32/pgalloc.h b/include/asm-avr32/pgalloc.h
index b77e364b4c444a164c9741d2d6ab94df1dfc2e35..51fc1f6e4b17afb9d73b7065ad6bea459549d205 100644
--- a/include/asm-avr32/pgalloc.h
+++ b/include/asm-avr32/pgalloc.h
@@ -17,10 +17,11 @@
set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte)))
static __inline__ void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
- struct page *pte)
+ pgtable_t pte)
{
set_pmd(pmd, __pmd(_PAGE_TABLE + page_to_phys(pte)));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* Allocate and free page tables
@@ -51,7 +52,9 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
struct page *pte;
pte = alloc_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
-
+ if (!pte)
+ return NULL;
+ pgtable_page_ctor(pte);
return pte;
}
@@ -60,12 +63,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
-#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) \
+do { \
+ pgtable_page_dtor(pte); \
+ tlb_remove_page((tlb), pte); \
+} while (0)
#define check_pgt_cache() do { } while(0)
diff --git a/include/asm-cris/page.h b/include/asm-cris/page.h
index 3b0156c4631107dc91dc13dd86468f2dff2c91ca..c45bb1ef397cb3466c2d836f8fc1679ba69d4a9d 100644
--- a/include/asm-cris/page.h
+++ b/include/asm-cris/page.h
@@ -26,6 +26,7 @@
typedef struct { unsigned long pte; } pte_t;
typedef struct { unsigned long pgd; } pgd_t;
typedef struct { unsigned long pgprot; } pgprot_t;
+typedef struct page *pgtable_t;
#endif
#define pte_val(x) ((x).pte)
diff --git a/include/asm-cris/pgalloc.h b/include/asm-cris/pgalloc.h
index 8ddd66f8177346aeba8c6dd42bf1d313e2355b0a..a1ba761d0573637ffb79ff4f9d6624d644dad9ce 100644
--- a/include/asm-cris/pgalloc.h
+++ b/include/asm-cris/pgalloc.h
@@ -6,6 +6,7 @@
#define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, pte)
#define pmd_populate(mm, pmd, pte) pmd_set(pmd, page_address(pte))
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* Allocate and free page tables.
@@ -27,10 +28,11 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long ad
return pte;
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *pte;
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
+ pgtable_page_ctor(pte);
return pte;
}
@@ -39,13 +41,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
-
-#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) \
+do { \
+ pgtable_page_dtor(pte); \
+ tlb_remove_page((tlb), pte); \
+} while (0)
#define check_pgt_cache() do { } while (0)
diff --git a/include/asm-frv/page.h b/include/asm-frv/page.h
index cacc045700de98eb36f156738cb12b2c24f0ad36..c2c1e89e747d248d78c0a7535fe2b4f11740edb7 100644
--- a/include/asm-frv/page.h
+++ b/include/asm-frv/page.h
@@ -25,6 +25,7 @@ typedef struct { unsigned long ste[64];} pmd_t;
typedef struct { pmd_t pue[1]; } pud_t;
typedef struct { pud_t pge[1]; } pgd_t;
typedef struct { unsigned long pgprot; } pgprot_t;
+typedef struct page *pgtable_t;
#define pte_val(x) ((x).pte)
#define pmd_val(x) ((x).ste[0])
diff --git a/include/asm-frv/pgalloc.h b/include/asm-frv/pgalloc.h
index e89620ef08cab876e315ec845d6c96d0d6e1cc97..971e6addb0095a4175d86b68715937e918d2f474 100644
--- a/include/asm-frv/pgalloc.h
+++ b/include/asm-frv/pgalloc.h
@@ -25,6 +25,7 @@
do { \
__set_pmd((PMD), page_to_pfn(PAGE) << PAGE_SHIFT | _PAGE_TABLE); \
} while(0)
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* Allocate and free page tables.
@@ -35,19 +36,24 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *);
extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long);
-extern struct page *pte_alloc_one(struct mm_struct *, unsigned long);
+extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long);
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
free_page((unsigned long)pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
-#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) \
+do { \
+ pgtable_page_dtor(pte); \
+ tlb_remove_page((tlb),(pte)); \
+} while (0)
/*
* allocating and freeing a pmd is trivial: the 1-entry pmd is
diff --git a/include/asm-ia64/page.h b/include/asm-ia64/page.h
index 8a8aa3fd7cd4493dfa6060418bed502cf1bd5227..4999a6c63775b87f4432378c6774e74f2ae2bfb0 100644
--- a/include/asm-ia64/page.h
+++ b/include/asm-ia64/page.h
@@ -185,6 +185,7 @@ get_order (unsigned long size)
#endif
typedef struct { unsigned long pgd; } pgd_t;
typedef struct { unsigned long pgprot; } pgprot_t;
+ typedef struct page *pgtable_t;
# define pte_val(x) ((x).pte)
# define pmd_val(x) ((x).pmd)
@@ -206,6 +207,7 @@ get_order (unsigned long size)
typedef unsigned long pmd_t;
typedef unsigned long pgd_t;
typedef unsigned long pgprot_t;
+ typedef struct page *pgtable_t;
# endif
# define pte_val(x) (x)
diff --git a/include/asm-ia64/pgalloc.h b/include/asm-ia64/pgalloc.h
index 556d988123ac1cd5ba351396caf39479401450b0..b9ac1a6fc21694a55e2771e471ab8cc90ab997cf 100644
--- a/include/asm-ia64/pgalloc.h
+++ b/include/asm-ia64/pgalloc.h
@@ -70,10 +70,11 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
#define __pmd_free_tlb(tlb, pmd) pmd_free((tlb)->mm, pmd)
static inline void
-pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, struct page *pte)
+pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, pgtable_t pte)
{
pmd_val(*pmd_entry) = page_to_phys(pte);
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
static inline void
pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte)
@@ -81,11 +82,17 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte)
pmd_val(*pmd_entry) = __pa(pte);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm,
- unsigned long addr)
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr)
{
- void *pg = quicklist_alloc(0, GFP_KERNEL, NULL);
- return pg ? virt_to_page(pg) : NULL;
+ struct page *page;
+ void *pg;
+
+ pg = quicklist_alloc(0, GFP_KERNEL, NULL);
+ if (!pg)
+ return NULL;
+ page = virt_to_page(pg);
+ pgtable_page_ctor(page);
+ return page;
}
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
@@ -94,8 +101,9 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
return quicklist_alloc(0, GFP_KERNEL, NULL);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
quicklist_free_page(0, NULL, pte);
}
diff --git a/include/asm-m32r/page.h b/include/asm-m32r/page.h
index 05d43bbbf940b022b140edb86256526ea338c620..8a677f3fca683d658dc41c6306ff6828f9df6390 100644
--- a/include/asm-m32r/page.h
+++ b/include/asm-m32r/page.h
@@ -28,6 +28,7 @@ typedef struct { unsigned long pgd; } pgd_t;
#define PTE_MASK PAGE_MASK
typedef struct { unsigned long pgprot; } pgprot_t;
+typedef struct page *pgtable_t;
#define pmd_val(x) ((x).pmd)
#define pgd_val(x) ((x).pgd)
diff --git a/include/asm-m32r/pgalloc.h b/include/asm-m32r/pgalloc.h
index e5921adfad1b6ae2490fb89aa61689f9eaf20b5e..f11a2b909cdbf8ab3bc1a11195b0160173c2a58b 100644
--- a/include/asm-m32r/pgalloc.h
+++ b/include/asm-m32r/pgalloc.h
@@ -9,10 +9,11 @@
set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte)))
static __inline__ void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
- struct page *pte)
+ pgtable_t pte)
{
set_pmd(pmd, __pmd(_PAGE_TABLE + page_to_phys(pte)));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* Allocate and free page tables.
@@ -37,12 +38,12 @@ static __inline__ pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
return pte;
}
-static __inline__ struct page *pte_alloc_one(struct mm_struct *mm,
+static __inline__ pgtable_t pte_alloc_one(struct mm_struct *mm,
unsigned long address)
{
struct page *pte = alloc_page(GFP_KERNEL|__GFP_ZERO);
-
+ pgtable_page_ctor(pte);
return pte;
}
@@ -51,8 +52,9 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
diff --git a/include/asm-m68k/motorola_pgalloc.h b/include/asm-m68k/motorola_pgalloc.h
index 500ec9b8b1892aebb41ec8833fa982abf5acaf9f..d08bf6261df88889f0e39091f8356580920208bb 100644
--- a/include/asm-m68k/motorola_pgalloc.h
+++ b/include/asm-m68k/motorola_pgalloc.h
@@ -7,7 +7,6 @@
extern pmd_t *get_pointer_table(void);
extern int free_pointer_table(pmd_t *);
-
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
pte_t *pte;
@@ -28,7 +27,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long) pte);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
pte_t *pte;
@@ -43,19 +42,21 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long add
nocache_page(pte);
}
kunmap(pte);
-
+ pgtable_page_ctor(page);
return page;
}
-static inline void pte_free(struct mm_struct *mm, struct page *page)
+static inline void pte_free(struct mm_struct *mm, pgtable_t page)
{
+ pgtable_page_dtor(page);
cache_page(kmap(page));
kunmap(page);
__free_page(page);
}
-static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *page)
+static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page)
{
+ pgtable_page_dtor(page);
cache_page(kmap(page));
kunmap(page);
__free_page(page);
@@ -94,10 +95,11 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *
pmd_set(pmd, pte);
}
-static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page)
+static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page)
{
pmd_set(pmd, page_address(page));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd)
{
diff --git a/include/asm-m68k/page.h b/include/asm-m68k/page.h
index 3f29e2a03a43bd203d34c633ed2c5a5749b35b4e..880c2cbff8a681cc8619ef50f5b26cd5fd202433 100644
--- a/include/asm-m68k/page.h
+++ b/include/asm-m68k/page.h
@@ -91,6 +91,7 @@ typedef struct { unsigned long pte; } pte_t;
typedef struct { unsigned long pmd[16]; } pmd_t;
typedef struct { unsigned long pgd; } pgd_t;
typedef struct { unsigned long pgprot; } pgprot_t;
+typedef struct page *pgtable_t;
#define pte_val(x) ((x).pte)
#define pmd_val(x) ((&x)->pmd[0])
diff --git a/include/asm-m68k/sun3_pgalloc.h b/include/asm-m68k/sun3_pgalloc.h
index a5a91e72714b0b9e715bba4ee0a008a777187057..d4c83f14381652402d35f4b3cb5fb5967f97a6de 100644
--- a/include/asm-m68k/sun3_pgalloc.h
+++ b/include/asm-m68k/sun3_pgalloc.h
@@ -26,12 +26,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long) pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *page)
+static inline void pte_free(struct mm_struct *mm, pgtable_t page)
{
+ pgtable_page_dtor(page);
__free_page(page);
}
-#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) \
+do { \
+ pgtable_page_dtor(pte); \
+ tlb_remove_page((tlb), pte); \
+} while (0)
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
unsigned long address)
@@ -45,8 +50,8 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
return (pte_t *) (page);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm,
- unsigned long address)
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
+ unsigned long address)
{
struct page *page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0);
@@ -54,6 +59,7 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
return NULL;
clear_highpage(page);
+ pgtable_page_ctor(page);
return page;
}
@@ -63,10 +69,11 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *
pmd_val(*pmd) = __pa((unsigned long)pte);
}
-static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page)
+static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page)
{
pmd_val(*pmd) = __pa((unsigned long)page_address(page));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* allocating and freeing a pmd is trivial: the 1-entry pmd is
diff --git a/include/asm-mips/page.h b/include/asm-mips/page.h
index 635aa44d2290dfcb85b056015341ff0580d232b3..8735aa0b89630987f9d7341efa88e13a4bf9eb7a 100644
--- a/include/asm-mips/page.h
+++ b/include/asm-mips/page.h
@@ -90,6 +90,7 @@ typedef struct { unsigned long pte; } pte_t;
#define pte_val(x) ((x).pte)
#define __pte(x) ((pte_t) { (x) } )
#endif
+typedef struct page *pgtable_t;
/*
* For 3-level pagetables we defines these ourselves, for 2-level the
diff --git a/include/asm-mips/pgalloc.h b/include/asm-mips/pgalloc.h
index c4efeced8396da58d0bad7b56fc4f2da50d2f753..1275831dda29b65ae90d2046f781a747cb4a3df0 100644
--- a/include/asm-mips/pgalloc.h
+++ b/include/asm-mips/pgalloc.h
@@ -20,10 +20,11 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
}
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
- struct page *pte)
+ pgtable_t pte)
{
set_pmd(pmd, __pmd((unsigned long)page_address(pte)));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* Initialize a new pmd table with invalid pointers.
@@ -79,9 +80,10 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
struct page *pte;
pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER);
- if (pte)
+ if (pte) {
clear_highpage(pte);
-
+ pgtable_page_ctor(pte);
+ }
return pte;
}
@@ -90,12 +92,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_pages((unsigned long)pte, PTE_ORDER);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_pages(pte, PTE_ORDER);
}
-#define __pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte))
+#define __pte_free_tlb(tlb,pte) \
+do { \
+ pgtable_page_dtor(pte); \
+ tlb_remove_page((tlb), pte); \
+} while (0)
#ifdef CONFIG_32BIT
diff --git a/include/asm-parisc/page.h b/include/asm-parisc/page.h
index b08d9151c71e4495b1ed9b8884199b92f7c007da..27d50b85954109d123a8e9118a8799a4eadc5221 100644
--- a/include/asm-parisc/page.h
+++ b/include/asm-parisc/page.h
@@ -91,6 +91,7 @@ typedef unsigned long pgprot_t;
#endif /* STRICT_MM_TYPECHECKS */
+typedef struct page *pgtable_t;
typedef struct __physmem_range {
unsigned long start_pfn;
diff --git a/include/asm-parisc/pgalloc.h b/include/asm-parisc/pgalloc.h
index aab66f1bea14d67c3381d39ba6bd0d15550467b7..3996dfc30a3f3366aebd714d63b35fb8455fcbac 100644
--- a/include/asm-parisc/pgalloc.h
+++ b/include/asm-parisc/pgalloc.h
@@ -115,11 +115,14 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte)
#define pmd_populate(mm, pmd, pte_page) \
pmd_populate_kernel(mm, pmd, page_address(pte_page))
+#define pmd_pgtable(pmd) pmd_page(pmd)
-static inline struct page *
+static inline pgtable_t
pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *page = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
+ if (page)
+ pgtable_page_ctor(page);
return page;
}
@@ -135,7 +138,11 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-#define pte_free(mm, page) pte_free_kernel(page_address(page))
+static inline void pte_free_kernel(struct mm_struct *mm, struct page *pte)
+{
+ pgtable_page_dtor(pte);
+ pte_free_kernel(page_address((pte));
+}
#define check_pgt_cache() do { } while (0)
diff --git a/include/asm-powerpc/page.h b/include/asm-powerpc/page.h
index 61e3725bbd37e18eae791ed2b3cf36e3d045b2eb..df47bbb6ea1348e6c6ad95898baa911e819823b7 100644
--- a/include/asm-powerpc/page.h
+++ b/include/asm-powerpc/page.h
@@ -190,6 +190,8 @@ extern int page_is_ram(unsigned long pfn);
struct vm_area_struct;
+typedef struct page *pgtable_t;
+
#include <asm-generic/memory_model.h>
#endif /* __ASSEMBLY__ */
diff --git a/include/asm-powerpc/pgalloc-32.h b/include/asm-powerpc/pgalloc-32.h
index c162a4c37b39ce5df12573c209dfd3a1edbdd59b..58c07147b3ea445c17851c8cb5ab8489a6432805 100644
--- a/include/asm-powerpc/pgalloc-32.h
+++ b/include/asm-powerpc/pgalloc-32.h
@@ -22,17 +22,19 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
(pmd_val(*(pmd)) = __pa(pte) | _PMD_PRESENT)
#define pmd_populate(mm, pmd, pte) \
(pmd_val(*(pmd)) = (page_to_pfn(pte) << PAGE_SHIFT) | _PMD_PRESENT)
+#define pmd_pgtable(pmd) pmd_page(pmd)
#else
#define pmd_populate_kernel(mm, pmd, pte) \
(pmd_val(*(pmd)) = (unsigned long)pte | _PMD_PRESENT)
#define pmd_populate(mm, pmd, pte) \
(pmd_val(*(pmd)) = (unsigned long)lowmem_page_address(pte) | _PMD_PRESENT)
+#define pmd_pgtable(pmd) pmd_page(pmd)
#endif
extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr);
-extern struct page *pte_alloc_one(struct mm_struct *mm, unsigned long addr);
+extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr);
extern void pte_free_kernel(struct mm_struct *mm, pte_t *pte);
-extern void pte_free(struct mm_struct *mm, struct page *pte);
+extern void pte_free(struct mm_struct *mm, pgtable_t pte);
#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, (pte))
diff --git a/include/asm-powerpc/pgalloc-64.h b/include/asm-powerpc/pgalloc-64.h
index 5afae8593931e5cf4966688307488adfc856a38b..68980990f62a764f3896cfe95830d412f98cf824 100644
--- a/include/asm-powerpc/pgalloc-64.h
+++ b/include/asm-powerpc/pgalloc-64.h
@@ -58,6 +58,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
#define pmd_populate(mm, pmd, pte_page) \
pmd_populate_kernel(mm, pmd, page_address(pte_page))
#define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte))
+#define pmd_pgtable(pmd) pmd_page(pmd)
#else /* CONFIG_PPC_64K_PAGES */
@@ -72,6 +73,7 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
#define pmd_populate(mm, pmd, pte_page) \
pmd_populate_kernel(mm, pmd, page_address(pte_page))
+#define pmd_pgtable(pmd) pmd_page(pmd)
#endif /* CONFIG_PPC_64K_PAGES */
@@ -92,11 +94,18 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm,
- unsigned long address)
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
+ unsigned long address)
{
- pte_t *pte = pte_alloc_one_kernel(mm, address);
- return pte ? virt_to_page(pte) : NULL;
+ struct page *page;
+ pte_t *pte;
+
+ pte = pte_alloc_one_kernel(mm, address);
+ if (!pte)
+ return NULL;
+ page = virt_to_page(pte);
+ pgtable_page_ctor(page);
+ return page;
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
@@ -104,8 +113,9 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *ptepage)
+static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
{
+ pgtable_page_dtor(ptepage);
__free_page(ptepage);
}
@@ -136,9 +146,12 @@ static inline void pgtable_free(pgtable_free_t pgf)
extern void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf);
-#define __pte_free_tlb(tlb, ptepage) \
+#define __pte_free_tlb(tlb,ptepage) \
+do { \
+ pgtable_page_dtor(ptepage); \
pgtable_free_tlb(tlb, pgtable_free_cache(page_address(ptepage), \
- PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1))
+ PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \
+} while (0)
#define __pmd_free_tlb(tlb, pmd) \
pgtable_free_tlb(tlb, pgtable_free_cache(pmd, \
PMD_CACHE_NUM, PMD_TABLE_SIZE-1))
diff --git a/include/asm-ppc/pgalloc.h b/include/asm-ppc/pgalloc.h
index 7c39a95829c7263932cf445fe94200c050991df4..fd4d1d74cfb1da92f1714e30755ef228ba3fb275 100644
--- a/include/asm-ppc/pgalloc.h
+++ b/include/asm-ppc/pgalloc.h
@@ -23,17 +23,19 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
(pmd_val(*(pmd)) = __pa(pte) | _PMD_PRESENT)
#define pmd_populate(mm, pmd, pte) \
(pmd_val(*(pmd)) = (page_to_pfn(pte) << PAGE_SHIFT) | _PMD_PRESENT)
+#define pmd_pgtable(pmd) pmd_page(pmd)
#else
#define pmd_populate_kernel(mm, pmd, pte) \
(pmd_val(*(pmd)) = (unsigned long)pte | _PMD_PRESENT)
#define pmd_populate(mm, pmd, pte) \
(pmd_val(*(pmd)) = (unsigned long)lowmem_page_address(pte) | _PMD_PRESENT)
+#define pmd_pgtable(pmd) pmd_page(pmd)
#endif
extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr);
-extern struct page *pte_alloc_one(struct mm_struct *mm, unsigned long addr);
+extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr);
extern void pte_free_kernel(struct mm_struct *mm, pte_t *pte);
-extern void pte_free(struct mm_struct *mm, struct page *pte);
+extern void pte_free(struct mm_struct *mm, pgtable_t pte);
#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, (pte))
diff --git a/include/asm-s390/page.h b/include/asm-s390/page.h
index a55f9d979dfb0a29aa87d8b61c2d1680529bc31c..7f29a981f48c78580bc80060bc72c798e7599ce7 100644
--- a/include/asm-s390/page.h
+++ b/include/asm-s390/page.h
@@ -109,6 +109,8 @@ typedef struct { unsigned long pgd; } pgd_t;
#endif /* __s390x__ */
+typedef struct page *pgtable_t;
+
#define __pte(x) ((pte_t) { (x) } )
#define __pmd(x) ((pmd_t) { (x) } )
#define __pgd(x) ((pgd_t) { (x) } )
diff --git a/include/asm-s390/pgalloc.h b/include/asm-s390/pgalloc.h
index 6f6619ba898062c6af642c83dd574eda02c0e6c6..900d44807e107bf2d8ce042b433cad88bf50f7e3 100644
--- a/include/asm-s390/pgalloc.h
+++ b/include/asm-s390/pgalloc.h
@@ -132,7 +132,7 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte)
}
static inline void
-pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page)
+pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page)
{
pte_t *pte = (pte_t *)page_to_phys(page);
pmd_t *shadow_pmd = get_shadow_table(pmd);
@@ -142,6 +142,7 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *page)
if (shadow_pmd && shadow_pte)
pmd_populate_kernel(mm, shadow_pmd, shadow_pte);
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* page table entry allocation/free routines.
diff --git a/include/asm-s390/tlb.h b/include/asm-s390/tlb.h
index 985de2b88279c99a563d85635af1ccec07fc0a6c..3c8177fa9e06e9b0b7800f14ce112917b9843dcc 100644
--- a/include/asm-s390/tlb.h
+++ b/include/asm-s390/tlb.h
@@ -95,7 +95,7 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
* pte_free_tlb frees a pte table and clears the CRSTE for the
* page table from the tlb.
*/
-static inline void pte_free_tlb(struct mmu_gather *tlb, struct page *page)
+static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t page)
{
if (!tlb->fullmm) {
tlb->array[tlb->nr_ptes++] = page;
diff --git a/include/asm-sh/page.h b/include/asm-sh/page.h
index e0fe02950f522b067d75bd8cf4f2d5ecbfeaf41c..134562dc8c4547224739ef99c819c91c39d4659c 100644
--- a/include/asm-sh/page.h
+++ b/include/asm-sh/page.h
@@ -100,6 +100,8 @@ typedef struct { unsigned long pgd; } pgd_t;
#define __pgd(x) ((pgd_t) { (x) } )
#define __pgprot(x) ((pgprot_t) { (x) } )
+typedef struct page *pgtable_t;
+
#endif /* !__ASSEMBLY__ */
/*
diff --git a/include/asm-sh/pgalloc.h b/include/asm-sh/pgalloc.h
index 59ca16d77a1d79b20775362e8d76d7d9cc2d3594..84dd2db7104c74f4f57eb4f1c7d0bfc1f4159869 100644
--- a/include/asm-sh/pgalloc.h
+++ b/include/asm-sh/pgalloc.h
@@ -14,10 +14,11 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
}
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
- struct page *pte)
+ pgtable_t pte)
{
set_pmd(pmd, __pmd((unsigned long)page_address(pte)));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
static inline void pgd_ctor(void *x)
{
@@ -47,11 +48,18 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
return quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm,
- unsigned long address)
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
+ unsigned long address)
{
- void *pg = quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL);
- return pg ? virt_to_page(pg) : NULL;
+ struct page *page;
+ void *pg;
+
+ pg = quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL);
+ if (!pg)
+ return NULL;
+ page = virt_to_page(pg);
+ pgtable_page_ctor(page);
+ return page;
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
@@ -59,12 +67,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
quicklist_free(QUICK_PT, NULL, pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
quicklist_free_page(QUICK_PT, NULL, pte);
}
-#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) \
+do { \
+ pgtable_page_dtor(pte); \
+ tlb_remove_page((tlb), (pte)); \
+} while (0)
/*
* allocating and freeing a pmd is trivial: the 1-entry pmd is
diff --git a/include/asm-sparc/page.h b/include/asm-sparc/page.h
index cbc48c0c4e15324fdafe23640036a5127315eea1..39ccf2da297c757e6b9e47f60d3dddec04bb4da4 100644
--- a/include/asm-sparc/page.h
+++ b/include/asm-sparc/page.h
@@ -123,6 +123,8 @@ typedef unsigned long iopgprot_t;
#endif
+typedef struct page *pgtable_t;
+
extern unsigned long sparc_unmapped_base;
BTFIXUPDEF_SETHI(sparc_unmapped_base)
diff --git a/include/asm-sparc/pgalloc.h b/include/asm-sparc/pgalloc.h
index b5fbdd36447fbf2b69f384957b76962731553645..6292cd00e5affdd8f913ddbfc1dbf290795384ac 100644
--- a/include/asm-sparc/pgalloc.h
+++ b/include/asm-sparc/pgalloc.h
@@ -50,10 +50,11 @@ BTFIXUPDEF_CALL(void, free_pmd_fast, pmd_t *)
BTFIXUPDEF_CALL(void, pmd_populate, pmd_t *, struct page *)
#define pmd_populate(MM, PMD, PTE) BTFIXUP_CALL(pmd_populate)(PMD, PTE)
+#define pmd_pgtable(pmd) pmd_page(pmd)
BTFIXUPDEF_CALL(void, pmd_set, pmd_t *, pte_t *)
#define pmd_populate_kernel(MM, PMD, PTE) BTFIXUP_CALL(pmd_set)(PMD, PTE)
-BTFIXUPDEF_CALL(struct page *, pte_alloc_one, struct mm_struct *, unsigned long)
+BTFIXUPDEF_CALL(pgtable_t , pte_alloc_one, struct mm_struct *, unsigned long)
#define pte_alloc_one(mm, address) BTFIXUP_CALL(pte_alloc_one)(mm, address)
BTFIXUPDEF_CALL(pte_t *, pte_alloc_one_kernel, struct mm_struct *, unsigned long)
#define pte_alloc_one_kernel(mm, addr) BTFIXUP_CALL(pte_alloc_one_kernel)(mm, addr)
@@ -61,7 +62,7 @@ BTFIXUPDEF_CALL(pte_t *, pte_alloc_one_kernel, struct mm_struct *, unsigned long
BTFIXUPDEF_CALL(void, free_pte_fast, pte_t *)
#define pte_free_kernel(mm, pte) BTFIXUP_CALL(free_pte_fast)(pte)
-BTFIXUPDEF_CALL(void, pte_free, struct page *)
+BTFIXUPDEF_CALL(void, pte_free, pgtable_t )
#define pte_free(mm, pte) BTFIXUP_CALL(pte_free)(pte)
#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, pte)
diff --git a/include/asm-sparc64/page.h b/include/asm-sparc64/page.h
index cdf950e017eeebad8cb7e553953ac8808ec188ec..e93a482aa24a0b874c50a2d109bf55f6b9c28fdc 100644
--- a/include/asm-sparc64/page.h
+++ b/include/asm-sparc64/page.h
@@ -104,6 +104,8 @@ typedef unsigned long pgprot_t;
#endif /* (STRICT_MM_TYPECHECKS) */
+typedef struct page *pgtable_t;
+
#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \
(_AC(0x0000000070000000,UL)) : \
(_AC(0xfffff80000000000,UL) + (1UL << 32UL)))
diff --git a/include/asm-sparc64/pgalloc.h b/include/asm-sparc64/pgalloc.h
index b48f73c2274ef7ee682204deffa9bcd09dad962f..3ee2d406373b9fc03a671d2c28b49adbb91dd634 100644
--- a/include/asm-sparc64/pgalloc.h
+++ b/include/asm-sparc64/pgalloc.h
@@ -43,11 +43,18 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
return quicklist_alloc(0, GFP_KERNEL, NULL);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm,
- unsigned long address)
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
+ unsigned long address)
{
- void *pg = quicklist_alloc(0, GFP_KERNEL, NULL);
- return pg ? virt_to_page(pg) : NULL;
+ struct page *page;
+ void *pg;
+
+ pg = quicklist_alloc(0, GFP_KERNEL, NULL);
+ if (!pg)
+ return NULL;
+ page = virt_to_page(pg);
+ pgtable_page_ctor(page);
+ return page;
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
@@ -55,8 +62,9 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
quicklist_free(0, NULL, pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *ptepage)
+static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
{
+ pgtable_page_dtor(ptepage);
quicklist_free_page(0, NULL, ptepage);
}
@@ -64,6 +72,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *ptepage)
#define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE)
#define pmd_populate(MM,PMD,PTE_PAGE) \
pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE))
+#define pmd_pgtable(pmd) pmd_page(pmd)
static inline void check_pgt_cache(void)
{
diff --git a/include/asm-um/page.h b/include/asm-um/page.h
index fe2374d705d15af36114379cfa5e4192a1125aca..381f96b1c8259153fca3433c9eca220031fa1e63 100644
--- a/include/asm-um/page.h
+++ b/include/asm-um/page.h
@@ -79,6 +79,8 @@ typedef unsigned long phys_t;
typedef struct { unsigned long pgprot; } pgprot_t;
+typedef struct page *pgtable_t;
+
#define pgd_val(x) ((x).pgd)
#define pgprot_val(x) ((x).pgprot)
diff --git a/include/asm-um/pgalloc.h b/include/asm-um/pgalloc.h
index 4f3e62b02861f0a743a3ffe45f280e127b0c7f01..9062a6e72241f60235b0c6f4c2bd41dd9c00ab6d 100644
--- a/include/asm-um/pgalloc.h
+++ b/include/asm-um/pgalloc.h
@@ -18,6 +18,7 @@
set_pmd(pmd, __pmd(_PAGE_TABLE + \
((unsigned long long)page_to_pfn(pte) << \
(unsigned long long) PAGE_SHIFT)))
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* Allocate and free page tables.
@@ -26,19 +27,24 @@ extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long);
-extern struct page *pte_alloc_one(struct mm_struct *, unsigned long);
+extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long);
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
free_page((unsigned long) pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
-#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) \
+do { \
+ pgtable_page_dtor(pte); \
+ tlb_remove_page((tlb),(pte)); \
+} while (0)
#ifdef CONFIG_3_LEVEL_PGTABLES
diff --git a/include/asm-x86/page_32.h b/include/asm-x86/page_32.h
index a6fd10f230d2e628660e252bda6623bc3840f67f..ba715d9798b093aeee1a50aecf5373caf8930835 100644
--- a/include/asm-x86/page_32.h
+++ b/include/asm-x86/page_32.h
@@ -50,6 +50,8 @@ typedef unsigned long phys_addr_t;
typedef union { pteval_t pte, pte_low; } pte_t;
typedef pte_t boot_pte_t;
+typedef struct page *pgtable_t;
+
#endif /* __ASSEMBLY__ */
#endif /* CONFIG_X86_PAE */
diff --git a/include/asm-x86/page_64.h b/include/asm-x86/page_64.h
index dcf0c0746075ba07f4527f2018bb846f8ebf8424..f7393bc516eff1c1d25ad799f0441f734a151280 100644
--- a/include/asm-x86/page_64.h
+++ b/include/asm-x86/page_64.h
@@ -71,6 +71,8 @@ typedef unsigned long pgdval_t;
typedef unsigned long pgprotval_t;
typedef unsigned long phys_addr_t;
+typedef struct page *pgtable_t;
+
typedef struct { pteval_t pte; } pte_t;
#define vmemmap ((struct page *)VMEMMAP_START)
diff --git a/include/asm-x86/pgalloc_32.h b/include/asm-x86/pgalloc_32.h
index bab12718a913fa446f25c8c9e6e2b2d380e3761a..6bea6e5b5ee55044dd06bea1b55677bf9cba20bf 100644
--- a/include/asm-x86/pgalloc_32.h
+++ b/include/asm-x86/pgalloc_32.h
@@ -31,6 +31,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *p
paravirt_alloc_pt(mm, pfn);
set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* Allocate and free page tables.
@@ -39,15 +40,16 @@ extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long);
-extern struct page *pte_alloc_one(struct mm_struct *, unsigned long);
+extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long);
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
free_page((unsigned long)pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
diff --git a/include/asm-x86/pgalloc_64.h b/include/asm-x86/pgalloc_64.h
index 4f6220db22b110bbf1606d75316a36e1b458fa1a..8d6722320dccd7818383abe4445eef5b9bc85f0c 100644
--- a/include/asm-x86/pgalloc_64.h
+++ b/include/asm-x86/pgalloc_64.h
@@ -12,6 +12,8 @@
#define pgd_populate(mm, pgd, pud) \
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)))
+#define pmd_pgtable(pmd) pmd_page(pmd)
+
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
{
set_pmd(pmd, __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT)));
@@ -91,12 +93,17 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long ad
return (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
- void *p = (void *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
+ struct page *page;
+ void *p;
+
+ p = (void *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
if (!p)
return NULL;
- return virt_to_page(p);
+ page = virt_to_page(p);
+ pgtable_page_ctor(page);
+ return page;
}
/* Should really implement gc for free page table pages. This could be
@@ -108,12 +115,17 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
free_page((unsigned long)pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *pte)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
-#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) \
+do { \
+ pgtable_page_dtor((pte)); \
+ tlb_remove_page((tlb), (pte)); \
+} while (0)
#define __pmd_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x))
#define __pud_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x))
diff --git a/include/asm-xtensa/page.h b/include/asm-xtensa/page.h
index 1adedbf41d019acd775bd1c8b3b61f664d649979..80a6ae0dd25957e548d53b4c3e125f32df042ba9 100644
--- a/include/asm-xtensa/page.h
+++ b/include/asm-xtensa/page.h
@@ -98,6 +98,7 @@
typedef struct { unsigned long pte; } pte_t; /* page table entry */
typedef struct { unsigned long pgd; } pgd_t; /* PGD table entry */
typedef struct { unsigned long pgprot; } pgprot_t;
+typedef struct page *pgtable_t;
#define pte_val(x) ((x).pte)
#define pgd_val(x) ((x).pgd)
diff --git a/include/asm-xtensa/pgalloc.h b/include/asm-xtensa/pgalloc.h
index 1d51ba5463f9f2f215015c598b5ef51bb7ff06f2..8d1544eb461ebfa13b3cd352786ba2af6fcf077e 100644
--- a/include/asm-xtensa/pgalloc.h
+++ b/include/asm-xtensa/pgalloc.h
@@ -24,6 +24,7 @@
(pmd_val(*(pmdp)) = ((unsigned long)ptep))
#define pmd_populate(mm, pmdp, page) \
(pmd_val(*(pmdp)) = ((unsigned long)page_to_virt(page)))
+#define pmd_pgtable(pmd) pmd_page(pmd)
static inline pgd_t*
pgd_alloc(struct mm_struct *mm)
@@ -46,10 +47,14 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
return kmem_cache_alloc(pgtable_cache, GFP_KERNEL|__GFP_REPEAT);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm,
- unsigned long addr)
+static inline pte_token_t pte_alloc_one(struct mm_struct *mm,
+ unsigned long addr)
{
- return virt_to_page(pte_alloc_one_kernel(mm, addr));
+ struct page *page;
+
+ page = virt_to_page(pte_alloc_one_kernel(mm, addr));
+ pgtable_page_ctor(page);
+ return page;
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
@@ -57,10 +62,12 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
kmem_cache_free(pgtable_cache, pte);
}
-static inline void pte_free(struct mm_struct *mm, struct page *page)
+static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
- kmem_cache_free(pgtable_cache, page_address(page));
+ pgtable_page_dtor(pte);
+ kmem_cache_free(pgtable_cache, page_address(pte));
}
+#define pmd_pgtable(pmd) pmd_page(pmd)
#endif /* __KERNEL__ */
#endif /* _XTENSA_PGALLOC_H */
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 89d7c691b93a6a660ee5468ac333a805b049a6f7..e8abb3814209f556a06750af5ba3941ed0796eda 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -894,6 +894,18 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a
#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
#endif /* NR_CPUS < CONFIG_SPLIT_PTLOCK_CPUS */
+static inline void pgtable_page_ctor(struct page *page)
+{
+ pte_lock_init(page);
+ inc_zone_page_state(page, NR_PAGETABLE);
+}
+
+static inline void pgtable_page_dtor(struct page *page)
+{
+ pte_lock_deinit(page);
+ dec_zone_page_state(page, NR_PAGETABLE);
+}
+
#define pte_offset_map_lock(mm, pmd, address, ptlp) \
({ \
spinlock_t *__ptl = pte_lockptr(mm, pmd); \
@@ -1136,7 +1148,7 @@ struct page *follow_page(struct vm_area_struct *, unsigned long address,
#define FOLL_GET 0x04 /* do get_page on page */
#define FOLL_ANON 0x08 /* give ZERO_PAGE if no pgtable */
-typedef int (*pte_fn_t)(pte_t *pte, struct page *pmd_page, unsigned long addr,
+typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
void *data);
extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
unsigned long size, pte_fn_t fn, void *data);
diff --git a/mm/memory.c b/mm/memory.c
index 153a54b2013ca9927edf1dda4e047a82a952c05e..e5628a5fd678ff2b8faa02521e49cec4d769970e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -134,11 +134,9 @@ void pmd_clear_bad(pmd_t *pmd)
*/
static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd)
{
- struct page *page = pmd_page(*pmd);
+ pgtable_t token = pmd_pgtable(*pmd);
pmd_clear(pmd);
- pte_lock_deinit(page);
- pte_free_tlb(tlb, page);
- dec_zone_page_state(page, NR_PAGETABLE);
+ pte_free_tlb(tlb, token);
tlb->mm->nr_ptes--;
}
@@ -309,21 +307,19 @@ void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
{
- struct page *new = pte_alloc_one(mm, address);
+ pgtable_t new = pte_alloc_one(mm, address);
if (!new)
return -ENOMEM;
- pte_lock_init(new);
spin_lock(&mm->page_table_lock);
- if (pmd_present(*pmd)) { /* Another has populated it */
- pte_lock_deinit(new);
- pte_free(mm, new);
- } else {
+ if (!pmd_present(*pmd)) { /* Has another populated it ? */
mm->nr_ptes++;
- inc_zone_page_state(new, NR_PAGETABLE);
pmd_populate(mm, pmd, new);
+ new = NULL;
}
spin_unlock(&mm->page_table_lock);
+ if (new)
+ pte_free(mm, new);
return 0;
}
@@ -334,11 +330,13 @@ int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
return -ENOMEM;
spin_lock(&init_mm.page_table_lock);
- if (pmd_present(*pmd)) /* Another has populated it */
- pte_free_kernel(&init_mm, new);
- else
+ if (!pmd_present(*pmd)) { /* Has another populated it ? */
pmd_populate_kernel(&init_mm, pmd, new);
+ new = NULL;
+ }
spin_unlock(&init_mm.page_table_lock);
+ if (new)
+ pte_free_kernel(&init_mm, new);
return 0;
}
@@ -1390,7 +1388,7 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
{
pte_t *pte;
int err;
- struct page *pmd_page;
+ pgtable_t token;
spinlock_t *uninitialized_var(ptl);
pte = (mm == &init_mm) ?
@@ -1401,10 +1399,10 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
BUG_ON(pmd_huge(*pmd));
- pmd_page = pmd_page(*pmd);
+ token = pmd_pgtable(*pmd);
do {
- err = fn(pte, pmd_page, addr, data);
+ err = fn(pte, token, addr, data);
if (err)
break;
} while (pte++, addr += PAGE_SIZE, addr != end);
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 0536dde139d1e1f6f28fc93ccd05a9fc337e2c7e..950c0be9ca8154d7b119019e50be5dc6838faa5b 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -820,7 +820,7 @@ void __attribute__((weak)) vmalloc_sync_all(void)
}
-static int f(pte_t *pte, struct page *pmd_page, unsigned long addr, void *data)
+static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
{
/* apply_to_page_range() does all the hard work. */
return 0;