From bf50bab2b34483316162443587b8467952e07730 Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Wed, 8 Sep 2010 10:19:33 +0900 Subject: hugetlb: add allocate function for hugepage migration We can't use existing hugepage allocation functions to allocate hugepage for page migration, because page migration can happen asynchronously with the running processes and page migration users should call the allocation function with physical addresses (not virtual addresses) as arguments. ChangeLog since v3: - unify alloc_buddy_huge_page() and alloc_buddy_huge_page_node() ChangeLog since v2: - remove unnecessary get/put_mems_allowed() (thanks to David Rientjes) ChangeLog since v1: - add comment on top of alloc_huge_page_no_vma() Signed-off-by: Naoya Horiguchi Acked-by: Mel Gorman Signed-off-by: Jun'ichi Nomura Reviewed-by: Christoph Lameter Signed-off-by: Andi Kleen --- include/linux/hugetlb.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index f479700..0b73c53 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -228,6 +228,8 @@ struct huge_bootmem_page { struct hstate *hstate; }; +struct page *alloc_huge_page_node(struct hstate *h, int nid); + /* arch callback */ int __init alloc_bootmem_huge_page(struct hstate *h); @@ -303,6 +305,7 @@ static inline struct hstate *page_hstate(struct page *page) #else struct hstate {}; +#define alloc_huge_page_node(h, nid) NULL #define alloc_bootmem_huge_page(h) NULL #define hstate_file(f) NULL #define hstate_vma(v) NULL -- cgit v1.1 From 0ebabb416f585ace711769057422af4bbc9d1110 Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Wed, 8 Sep 2010 10:19:34 +0900 Subject: hugetlb: redefine hugepage copy functions This patch modifies hugepage copy functions to have only destination and source hugepages as arguments for later use. The old ones are renamed from copy_{gigantic,huge}_page() to copy_user_{gigantic,huge}_page(). This naming convention is consistent with that between copy_highpage() and copy_user_highpage(). ChangeLog since v4: - add blank line between local declaration and code - remove unnecessary might_sleep() ChangeLog since v2: - change copy_huge_page() from macro to inline dummy function to avoid compile warning when !CONFIG_HUGETLB_PAGE. Signed-off-by: Naoya Horiguchi Acked-by: Mel Gorman Reviewed-by: Christoph Lameter Signed-off-by: Andi Kleen --- include/linux/hugetlb.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 0b73c53..9e51f77 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -44,6 +44,7 @@ int hugetlb_reserve_pages(struct inode *inode, long from, long to, int acctflags); void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed); void __isolate_hwpoisoned_huge_page(struct page *page); +void copy_huge_page(struct page *dst, struct page *src); extern unsigned long hugepages_treat_as_movable; extern const unsigned long hugetlb_zero, hugetlb_infinity; @@ -102,6 +103,9 @@ static inline void hugetlb_report_meminfo(struct seq_file *m) #define hugetlb_fault(mm, vma, addr, flags) ({ BUG(); 0; }) #define huge_pte_offset(mm, address) 0 #define __isolate_hwpoisoned_huge_page(page) 0 +static inline void copy_huge_page(struct page *dst, struct page *src) +{ +} #define hugetlb_change_protection(vma, address, end, newprot) -- cgit v1.1 From 290408d4a25002f099efeee7b6a5778d431154d6 Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Wed, 8 Sep 2010 10:19:35 +0900 Subject: hugetlb: hugepage migration core This patch extends page migration code to support hugepage migration. One of the potential users of this feature is soft offlining which is triggered by memory corrected errors (added by the next patch.) Todo: - there are other users of page migration such as memory policy, memory hotplug and memocy compaction. They are not ready for hugepage support for now. ChangeLog since v4: - define migrate_huge_pages() - remove changes on isolation/putback_lru_page() ChangeLog since v2: - refactor isolate/putback_lru_page() to handle hugepage - add comment about race on unmap_and_move_huge_page() ChangeLog since v1: - divide migration code path for hugepage - define routine checking migration swap entry for hugetlb - replace "goto" with "if/else" in remove_migration_pte() Signed-off-by: Naoya Horiguchi Signed-off-by: Jun'ichi Nomura Acked-by: Mel Gorman Signed-off-by: Andi Kleen --- include/linux/migrate.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'include/linux') diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 7238231..3c1941e 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -14,6 +14,8 @@ extern int migrate_page(struct address_space *, struct page *, struct page *); extern int migrate_pages(struct list_head *l, new_page_t x, unsigned long private, int offlining); +extern int migrate_huge_pages(struct list_head *l, new_page_t x, + unsigned long private, int offlining); extern int fail_migrate_page(struct address_space *, struct page *, struct page *); @@ -23,12 +25,17 @@ extern int migrate_prep_local(void); extern int migrate_vmas(struct mm_struct *mm, const nodemask_t *from, const nodemask_t *to, unsigned long flags); +extern void migrate_page_copy(struct page *newpage, struct page *page); +extern int migrate_huge_page_move_mapping(struct address_space *mapping, + struct page *newpage, struct page *page); #else #define PAGE_MIGRATION 0 static inline void putback_lru_pages(struct list_head *l) {} static inline int migrate_pages(struct list_head *l, new_page_t x, unsigned long private, int offlining) { return -ENOSYS; } +static inline int migrate_huge_pages(struct list_head *l, new_page_t x, + unsigned long private, int offlining) { return -ENOSYS; } static inline int migrate_prep(void) { return -ENOSYS; } static inline int migrate_prep_local(void) { return -ENOSYS; } @@ -40,6 +47,15 @@ static inline int migrate_vmas(struct mm_struct *mm, return -ENOSYS; } +static inline void migrate_page_copy(struct page *newpage, + struct page *page) {} + +extern int migrate_huge_page_move_mapping(struct address_space *mapping, + struct page *newpage, struct page *page) +{ + return -ENOSYS; +} + /* Possible settings for the migrate_page() method in address_operations */ #define migrate_page NULL #define fail_migrate_page NULL -- cgit v1.1 From 6de2b1aab94355482bd2accdc115666509667458 Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Wed, 8 Sep 2010 10:19:36 +0900 Subject: HWPOISON, hugetlb: add free check to dequeue_hwpoison_huge_page() This check is necessary to avoid race between dequeue and allocation, which can cause a free hugepage to be dequeued twice and get kernel unstable. Signed-off-by: Naoya Horiguchi Signed-off-by: Wu Fengguang Acked-by: Mel Gorman Reviewed-by: Christoph Lameter Signed-off-by: Andi Kleen --- include/linux/hugetlb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 9e51f77..796f30e 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -43,7 +43,7 @@ int hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_struct *vma, int acctflags); void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed); -void __isolate_hwpoisoned_huge_page(struct page *page); +int dequeue_hwpoisoned_huge_page(struct page *page); void copy_huge_page(struct page *dst, struct page *src); extern unsigned long hugepages_treat_as_movable; @@ -102,7 +102,7 @@ static inline void hugetlb_report_meminfo(struct seq_file *m) #define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; }) #define hugetlb_fault(mm, vma, addr, flags) ({ BUG(); 0; }) #define huge_pte_offset(mm, address) 0 -#define __isolate_hwpoisoned_huge_page(page) 0 +#define dequeue_hwpoisoned_huge_page(page) 0 static inline void copy_huge_page(struct page *dst, struct page *src) { } -- cgit v1.1 From 6f39ce056ab2ab2d29b2fae4aed61ed0b485972f Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Thu, 30 Sep 2010 11:54:51 +0900 Subject: Fix build error with !CONFIG_MIGRATION migrate_huge_page_move_mapping() is declared as "extern int ..." in include/linux/migrate.h for !CONFIG_MIGRATION, which causes the build error like below: mm/mprotect.o: In function `migrate_huge_page_move_mapping': mprotect.c:(.text+0x0): multiple definition of `migrate_huge_page_move_mapping' mm/shmem.o:shmem.c:(.text+0x0): first defined here mm/rmap.o: In function `migrate_huge_page_move_mapping': rmap.c:(.text+0x0): multiple definition of `migrate_huge_page_move_mapping' mm/shmem.o:shmem.c:(.text+0x0): first defined here Reported-by: Stephen Rothwell Signed-off-by: Naoya Horiguchi Signed-off-by: Andi Kleen --- include/linux/migrate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 3c1941e..085527f 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -50,7 +50,7 @@ static inline int migrate_vmas(struct mm_struct *mm, static inline void migrate_page_copy(struct page *newpage, struct page *page) {} -extern int migrate_huge_page_move_mapping(struct address_space *mapping, +static inline int migrate_huge_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page) { return -ENOSYS; -- cgit v1.1 From aa50d3a7aa8147b9e14dc9d5972a5d2359db4ef8 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Wed, 6 Oct 2010 21:45:00 +0200 Subject: Encode huge page size for VM_FAULT_HWPOISON errors This fixes a problem introduced with the hugetlb hwpoison handling The user space SIGBUS signalling wants to know the size of the hugepage that caused a HWPOISON fault. Unfortunately the architecture page fault handlers do not have easy access to the struct page. Pass the information out in the fault error code instead. I added a separate VM_FAULT_HWPOISON_LARGE bit for this case and encode the hpage index in some free upper bits of the fault code. The small page hwpoison keeps stays with the VM_FAULT_HWPOISON name to minimize changes. Also add code to hugetlb.h to convert that index into a page shift. Will be used in a further patch. Cc: Naoya Horiguchi Cc: fengguang.wu@intel.com Signed-off-by: Andi Kleen --- include/linux/hugetlb.h | 6 ++++++ include/linux/mm.h | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 796f30e..943c76b 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -307,6 +307,11 @@ static inline struct hstate *page_hstate(struct page *page) return size_to_hstate(PAGE_SIZE << compound_order(page)); } +static inline unsigned hstate_index_to_shift(unsigned index) +{ + return hstates[index].order + PAGE_SHIFT; +} + #else struct hstate {}; #define alloc_huge_page_node(h, nid) NULL @@ -324,6 +329,7 @@ static inline unsigned int pages_per_huge_page(struct hstate *h) { return 1; } +#define hstate_index_to_shift(index) 0 #endif #endif /* _LINUX_HUGETLB_H */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 74949fb..f7e9efc 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -718,12 +718,20 @@ static inline int page_mapped(struct page *page) #define VM_FAULT_SIGBUS 0x0002 #define VM_FAULT_MAJOR 0x0004 #define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */ -#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned page */ +#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */ +#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */ #define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */ #define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */ -#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON) +#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */ + +#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | \ + VM_FAULT_HWPOISON_LARGE) + +/* Encode hstate index for a hwpoisoned large page */ +#define VM_FAULT_SET_HINDEX(x) ((x) << 12) +#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf) /* * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. -- cgit v1.1