summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndrea Arcangeli <aarcange@redhat.com>2011-01-13 15:47:18 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2011-01-13 17:32:47 -0800
commit60ab3244ec85c44276c585a2a20d3750402e1cf4 (patch)
treee69e866b370243fc58a6fc721e5347a265e8fd4f
parenta664b2d8555c659127bf8fe049a58449d394a707 (diff)
downloadop-kernel-dev-60ab3244ec85c44276c585a2a20d3750402e1cf4.zip
op-kernel-dev-60ab3244ec85c44276c585a2a20d3750402e1cf4.tar.gz
thp: khugepaged: make khugepaged aware about madvise
MADV_HUGEPAGE and MADV_NOHUGEPAGE were fully effective only if run after mmap and before touching the memory. While this is enough for most usages, it's little effort to make madvise more dynamic at runtime on an existing mapping by making khugepaged aware about madvise. MADV_HUGEPAGE: register in khugepaged immediately without waiting a page fault (that may not ever happen if all pages are already mapped and the "enabled" knob was set to madvise during the initial page faults). MADV_NOHUGEPAGE: skip vmas marked VM_NOHUGEPAGE in khugepaged to stop collapsing pages where not needed. [akpm@linux-foundation.org: tweak comment] Signed-off-by: Andrea Arcangeli <aarcange@redhat.com> Cc: Michael Kerrisk <mtk.manpages@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r--include/linux/huge_mm.h6
-rw-r--r--mm/huge_memory.c23
-rw-r--r--mm/madvise.c2
3 files changed, 24 insertions, 7 deletions
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index a8b7e42..bddfba1 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -105,7 +105,8 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
#if HPAGE_PMD_ORDER > MAX_ORDER
#error "hugepages can't be allocated by the buddy allocator"
#endif
-extern int hugepage_madvise(unsigned long *vm_flags, int advice);
+extern int hugepage_madvise(struct vm_area_struct *vma,
+ unsigned long *vm_flags, int advice);
extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
unsigned long start,
unsigned long end,
@@ -143,7 +144,8 @@ static inline int split_huge_page(struct page *page)
do { } while (0)
#define wait_split_huge_page(__anon_vma, __pmd) \
do { } while (0)
-static inline int hugepage_madvise(unsigned long *vm_flags, int advice)
+static inline int hugepage_madvise(struct vm_area_struct *vma,
+ unsigned long *vm_flags, int advice)
{
BUG();
return 0;
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index fce667c0..004c9c2 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1389,7 +1389,8 @@ out:
return ret;
}
-int hugepage_madvise(unsigned long *vm_flags, int advice)
+int hugepage_madvise(struct vm_area_struct *vma,
+ unsigned long *vm_flags, int advice)
{
switch (advice) {
case MADV_HUGEPAGE:
@@ -1404,6 +1405,13 @@ int hugepage_madvise(unsigned long *vm_flags, int advice)
return -EINVAL;
*vm_flags &= ~VM_NOHUGEPAGE;
*vm_flags |= VM_HUGEPAGE;
+ /*
+ * If the vma become good for khugepaged to scan,
+ * register it here without waiting a page fault that
+ * may not happen any time soon.
+ */
+ if (unlikely(khugepaged_enter_vma_merge(vma)))
+ return -ENOMEM;
break;
case MADV_NOHUGEPAGE:
/*
@@ -1417,6 +1425,11 @@ int hugepage_madvise(unsigned long *vm_flags, int advice)
return -EINVAL;
*vm_flags &= ~VM_HUGEPAGE;
*vm_flags |= VM_NOHUGEPAGE;
+ /*
+ * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
+ * this vma even if we leave the mm registered in khugepaged if
+ * it got registered before VM_NOHUGEPAGE was set.
+ */
break;
}
@@ -1784,7 +1797,8 @@ static void collapse_huge_page(struct mm_struct *mm,
if (address < hstart || address + HPAGE_PMD_SIZE > hend)
goto out;
- if (!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always())
+ if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
+ (vma->vm_flags & VM_NOHUGEPAGE))
goto out;
/* VM_PFNMAP vmas may have vm_ops null but vm_file set */
@@ -2007,8 +2021,9 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
break;
}
- if (!(vma->vm_flags & VM_HUGEPAGE) &&
- !khugepaged_always()) {
+ if ((!(vma->vm_flags & VM_HUGEPAGE) &&
+ !khugepaged_always()) ||
+ (vma->vm_flags & VM_NOHUGEPAGE)) {
progress++;
continue;
}
diff --git a/mm/madvise.c b/mm/madvise.c
index bbac126..2221491 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -73,7 +73,7 @@ static long madvise_behavior(struct vm_area_struct * vma,
break;
case MADV_HUGEPAGE:
case MADV_NOHUGEPAGE:
- error = hugepage_madvise(&new_flags, behavior);
+ error = hugepage_madvise(vma, &new_flags, behavior);
if (error)
goto out;
break;
OpenPOWER on IntegriCloud