Per task vma cache. Signed-off-by: Peter Zijlstra --- include/linux/sched.h | 4 +- include/linux/vma_cache.h | 92 ++++++++++++++++++++++++++++++++++++++++++++++ kernel/fork.c | 1 mm/mmap.c | 64 +++++++++++++++++++++++++++++--- 4 files changed, 153 insertions(+), 8 deletions(-) Index: linux-2.6-rt/include/linux/sched.h =================================================================== --- linux-2.6-rt.orig/include/linux/sched.h +++ linux-2.6-rt/include/linux/sched.h @@ -83,6 +83,7 @@ struct sched_param { #include #include #include +#include #include @@ -467,7 +468,6 @@ typedef unsigned long mm_counter_t; struct mm_struct { struct vm_area_struct * mmap; /* list of VMAs */ struct rb_root mm_rb; - struct vm_area_struct * mmap_cache; /* last find_vma result */ unsigned long (*get_unmapped_area) (struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); @@ -1269,6 +1269,8 @@ struct task_struct { #ifdef CONFIG_FAULT_INJECTION int make_it_fail; #endif + + struct vma_cache vma_cache; }; static inline pid_t process_group(struct task_struct *tsk) Index: linux-2.6-rt/include/linux/vma_cache.h =================================================================== --- /dev/null +++ linux-2.6-rt/include/linux/vma_cache.h @@ -0,0 +1,92 @@ +#ifndef _LINUX_VMA_CACHE_H +#define _LINUX_VMA_CACHE_H + +#include +#include + +struct vma_cache_entry { + unsigned long vm_start; + unsigned long vm_end; + unsigned long vm_flags; + struct vm_area_struct *vma; +}; + +struct vma_cache { + unsigned seq; + unsigned hand; + struct vma_cache_entry entry[4]; +}; + +#define VM_CACHE_ACTIVE 0x80000000UL + +static inline void vma_cache_set_active(struct vma_cache_entry *entry) +{ + entry->vm_flags |= VM_CACHE_ACTIVE; +} + +static inline bool vma_cache_test_clear_active(struct vma_cache_entry *entry) +{ + bool active = entry->vm_flags & VM_CACHE_ACTIVE; + + entry->vm_flags &= ~VM_CACHE_ACTIVE; + + return active; +} + +static inline +bool vma_cache_entry_match(struct vma_cache_entry *vce, unsigned long addr) +{ + return vce->vm_start <= addr && vce->vm_end > addr; +} + +static inline +struct vma_cache_entry *vma_cache_find(struct vma_cache *vc, unsigned long addr) +{ + int i; + struct vma_cache_entry *tmp, *entry = NULL; + + for (i = 0; i < ARRAY_SIZE(vc->entry); i++) { + tmp = &vc->entry[i]; + if (vma_cache_entry_match(tmp, addr)) { + vma_cache_set_active(tmp); + entry = tmp; + break; + } + } + + return entry; +} + +static inline void vma_cache_blow(struct vma_cache *vc) +{ + memset(vc, 0, sizeof(struct vma_cache)); +} + +#define VMA_CACHE_ENTRY(vma) \ + (struct vma_cache_entry){ \ + .vm_start = (vma)->vm_start, \ + .vm_end = (vma)->vm_end, \ + .vm_flags = (vma)->vm_flags, \ + .vma = (vma) \ + } + +static inline +struct vma_cache_entry *vma_cache_new(struct vma_cache *vc) +{ + struct vma_cache_entry *entry; + + for (;;) { + vc->hand = (vc->hand + 1) % ARRAY_SIZE(vc->entry); + entry = &vc->entry[vc->hand]; + if (!vma_cache_test_clear_active(entry)) + return entry; + } + + BUG(); + + return NULL; +} + +extern bool vma_anon(unsigned long addr); + +#endif Index: linux-2.6-rt/mm/mmap.c =================================================================== --- linux-2.6-rt.orig/mm/mmap.c +++ linux-2.6-rt/mm/mmap.c @@ -473,8 +473,6 @@ __vma_unlink(struct mm_struct *mm, struc { prev->vm_next = vma->vm_next; rb_erase(&vma->vm_rb, &mm->mm_rb); - if (mm->mmap_cache == vma) - mm->mmap_cache = prev; } /* @@ -1397,6 +1395,62 @@ get_unmapped_area(struct file *file, uns EXPORT_SYMBOL(get_unmapped_area); +static bool vma_cache_valid(struct mm_struct *mm) +{ + unsigned seq = mm_sequence(mm); + + if (current->vma_cache.seq != seq) { + vma_cache_blow(¤t->vma_cache); + current->vma_cache.seq = seq; + return false; + } + + return true; +} + +/* + * when true the vma is an anonymous vma + * when false, it might still be. + */ +bool vma_anon(unsigned long addr) +{ + struct vma_cache_entry *entry = + vma_cache_find(¤t->vma_cache, addr); + + if (vma_cache_valid(current->mm) && entry) + return !(entry->vm_flags & VM_SHARED); + + return false; +} + +static +struct vm_area_struct *find_vma_cached(struct mm_struct *mm, unsigned long addr) +{ + struct vma_cache_entry *entry; + + if (current->mm != mm) + return NULL; + + entry = vma_cache_find(¤t->vma_cache, addr); + if (vma_cache_valid(mm) && entry) + return entry->vma; + + return NULL; +} + +static +void insert_vma_cache(struct mm_struct *mm, struct vm_area_struct *vma) +{ + struct vma_cache_entry *entry; + + if (current->mm != mm || (mm_sequence(mm) & 1)) + return; + + vma_cache_valid(mm); + entry = vma_cache_new(¤t->vma_cache); + *entry = VMA_CACHE_ENTRY(vma); +} + /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr) { @@ -1404,8 +1458,7 @@ struct vm_area_struct * find_vma(struct if (mm) { /* Check the cache first. */ - /* (Cache hit rate is typically around 35%.) */ - vma = mm->mmap_cache; + vma = find_vma_cached(mm, addr); if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) { struct rb_node * rb_node; @@ -1427,7 +1480,7 @@ struct vm_area_struct * find_vma(struct rb_node = rb_node->rb_right; } if (vma) - mm->mmap_cache = vma; + insert_vma_cache(mm, vma); } } return vma; @@ -1726,7 +1779,6 @@ detach_vmas_to_be_unmapped(struct mm_str else addr = vma ? vma->vm_start : mm->mmap_base; mm->unmap_area(mm, addr); - mm->mmap_cache = NULL; /* Kill the cache. */ } /* Index: linux-2.6-rt/kernel/fork.c =================================================================== --- linux-2.6-rt.orig/kernel/fork.c +++ linux-2.6-rt/kernel/fork.c @@ -248,7 +248,6 @@ static inline int dup_mmap(struct mm_str mm->locked_vm = 0; mm->mmap = NULL; - mm->mmap_cache = NULL; mm->free_area_cache = oldmm->mmap_base; mm->cached_hole_size = ~0UL; mm->map_count = 0;