--- include/linux/reserve.h | 95 ++++++++++++++++++++++++++++++ mm/reserve.c | 148 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+) Index: linux-2.6/include/linux/reserve.h =================================================================== --- linux-2.6.orig/include/linux/reserve.h +++ linux-2.6/include/linux/reserve.h @@ -11,6 +11,8 @@ #include #include +#include +#include struct mem_reserve { struct mem_reserve *parent; @@ -23,6 +25,8 @@ struct mem_reserve { long limit; long usage; spinlock_t lock; /* protects limit and usage */ + + wait_queue_head_t waitqueue; }; extern struct mem_reserve mem_reserve_root; @@ -51,4 +55,95 @@ int mem_reserve_kmem_cache_charge(struct struct kmem_cache *s, long objs, int overcommit); +void *___kmalloc_reserve(size_t size, gfp_t flags, int node, void *ip, + struct mem_reserve *res, int *emerg); + +static inline +void *__kmalloc_reserve(size_t size, gfp_t flags, int node, void *ip, + struct mem_reserve *res, int *emerg) +{ + void *obj; + + obj = __kmalloc_node_track_caller(size, + flags | __GFP_NOMEMALLOC | __GFP_NOWARN, node, ip); + if (!obj) + obj = ___kmalloc_reserve(size, flags, node, ip, res, emerg); + + return obj; +} + +#define kmalloc_reserve(size, gfp, node, res, emerg) \ + __kmalloc_reserve(size, gfp, node, \ + __builtin_return_address(0), res, emerg) + +void __kfree_reserve(void *obj, struct mem_reserve *res, int emerg); + +static inline +void kfree_reserve(void *obj, struct mem_reserve *res, int emerg) +{ + if (unlikely(obj && res && emerg)) + __kfree_reserve(obj, res, emerg); + else + kfree(obj); +} + +void *__kmem_cache_alloc_reserve(struct kmem_cache *s, gfp_t flags, int node, + struct mem_reserve *res, int *emerg); +static inline +void *kmem_cache_alloc_reserve(struct kmem_cache *s, gfp_t flags, int node, + struct mem_reserve *res, int *emerg) +{ + void *obj; + + obj = kmem_cache_alloc_node(s, + flags | __GFP_NOMEMALLOC | __GFP_NOWARN, node); + if (!obj) + obj = __kmem_cache_alloc_reserve(s, flags, node, res, emerg); + + return obj; +} + +void __kmem_cache_free_reserve(struct kmem_cache *s, void *obj, + struct mem_reserve *res, int emerg); + +static inline +void kmem_cache_free_reserve(struct kmem_cache *s, void *obj, + struct mem_reserve *res, int emerg) +{ + if (unlikely(obj && res && emerg)) + __kmem_cache_free_reserve(s, obj, res, emerg); + else + kmem_cache_free(s, obj); +} + +struct page *__alloc_pages_reserve(int node, gfp_t flags, int order, + struct mem_reserve *res, int *emerg); + +static inline +struct page *alloc_pages_reserve(int node, gfp_t flags, int order, + struct mem_reserve *res, int *emerg) +{ + struct page *page; + + page = alloc_pages_node(node, + flags | __GFP_NOMEMALLOC | __GFP_NOWARN, order); + if (!page) + page = __alloc_pages_reserve(node, flags, order, res, emerg); + + return page; +} + +void __free_pages_reserve(struct page *page, int order, + struct mem_reserve *res, int emerg); + +static inline +void free_pages_reserve(struct page *page, int order, + struct mem_reserve *res, int emerg) +{ + if (unlikely(page && res && emerg)) + __free_pages_reserve(page, order, res, emerg); + else + __free_pages(page, order); +} + #endif /* _LINUX_RESERVE_H */ Index: linux-2.6/mm/reserve.c =================================================================== --- linux-2.6.orig/mm/reserve.c +++ linux-2.6/mm/reserve.c @@ -39,6 +39,8 @@ #include #include #include +#include +#include "internal.h" static DEFINE_MUTEX(mem_reserve_mutex); @@ -54,6 +56,7 @@ struct mem_reserve mem_reserve_root = { .siblings = LIST_HEAD_INIT(mem_reserve_root.siblings), .name = "total reserve", .lock = __SPIN_LOCK_UNLOCKED(mem_reserve_root.lock), + .waitqueue = __WAIT_QUEUE_HEAD_INITIALIZER(mem_reserve_root.waitqueue), }; EXPORT_SYMBOL_GPL(mem_reserve_root); @@ -71,6 +74,7 @@ void mem_reserve_init(struct mem_reserve INIT_LIST_HEAD(&res->siblings); res->name = name; spin_lock_init(&res->lock); + init_waitqueue_head(&res->waitqueue); if (parent) mem_reserve_connect(res, parent); @@ -443,3 +447,147 @@ int mem_reserve_kmem_cache_charge(struct return __mem_reserve_charge(res, objs * kmem_cache_size(s), overcommit); } EXPORT_SYMBOL_GPL(mem_reserve_kmem_cache_charge); + +/* + * alloc wrappers + */ + +void *___kmalloc_reserve(size_t size, gfp_t flags, int node, void *ip, + struct mem_reserve *res, int *emerg) +{ + void *obj; + gfp_t gfp; + + gfp = flags | __GFP_NOMEMALLOC | __GFP_NOWARN; + obj = __kmalloc_node_track_caller(size, gfp, node, ip); + + if (obj || !(gfp_to_alloc_flags(flags) & ALLOC_NO_WATERMARKS)) + goto out; + + if (res && !mem_reserve_kmalloc_charge(res, size, 0)) { + if (!(flags & __GFP_WAIT)) + goto out; + + wait_event(res->waitqueue, + mem_reserve_kmalloc_charge(res, size, 0)); + + obj = __kmalloc_node_track_caller(size, gfp, node, ip); + if (obj) { + mem_reserve_kmalloc_charge(res, -size, 0); + wake_up_all(&res->waitqueue); + goto out; + } + } + + obj = __kmalloc_node_track_caller(size, flags, node, ip); + WARN_ON(!obj); + if (emerg) + *emerg |= 1; + +out: + return obj; +} + +void __kfree_reserve(void *obj, struct mem_reserve *res, int emerg) +{ + size_t size = ksize(obj); + + kfree(obj); + /* + * ksize gives the full allocated size vs the requested size we used to + * charge; however since we round up to the nearest power of two, this + * should all work nicely. + */ + mem_reserve_kmalloc_charge(res, -size, 0); + wake_up_all(&res->waitqueue); +} + + +void *__kmem_cache_alloc_reserve(struct kmem_cache *s, gfp_t flags, int node, + struct mem_reserve *res, int *emerg) +{ + void *obj; + gfp_t gfp; + + gfp = flags | __GFP_NOMEMALLOC | __GFP_NOWARN; + obj = kmem_cache_alloc_node(s, gfp, node); + + if (obj || !(gfp_to_alloc_flags(flags) & ALLOC_NO_WATERMARKS)) + goto out; + + if (res && !mem_reserve_kmem_cache_charge(res, s, 1, 0)) { + if (!(flags & __GFP_WAIT)) + goto out; + + wait_event(res->waitqueue, + mem_reserve_kmem_cache_charge(res, s, 1, 0)); + + obj = kmem_cache_alloc_node(s, gfp, node); + if (obj) { + mem_reserve_kmem_cache_charge(res, s, -1, 0); + wake_up_all(&res->waitqueue); + goto out; + } + } + + obj = kmem_cache_alloc_node(s, flags, node); + WARN_ON(!obj); + if (emerg) + *emerg |= 1; + +out: + return obj; +} + +void __kmem_cache_free_reserve(struct kmem_cache *s, void *obj, + struct mem_reserve *res, int emerg) +{ + kmem_cache_free(s, obj); + mem_reserve_kmem_cache_charge(res, s, -1, 0); + wake_up_all(&res->waitqueue); +} + + +struct page *__alloc_pages_reserve(int node, gfp_t flags, int order, + struct mem_reserve *res, int *emerg) +{ + struct page *page; + gfp_t gfp; + + gfp = flags | __GFP_NOMEMALLOC | __GFP_NOWARN; + page = alloc_pages_node(node, gfp, order); + + if (page || !(gfp_to_alloc_flags(flags) & ALLOC_NO_WATERMARKS)) + goto out; + + if (res && !mem_reserve_pages_charge(res, 1 << order, 0)) { + if (!(flags & __GFP_WAIT)) + goto out; + + wait_event(res->waitqueue, + mem_reserve_pages_charge(res, 1 << order, 0)); + + page = alloc_pages_node(node, gfp, order); + if (page) { + mem_reserve_pages_charge(res, -(1 << order), 0); + wake_up_all(&res->waitqueue); + goto out; + } + } + + page = alloc_pages_node(node, flags, order); + WARN_ON(!page); + if (emerg) + *emerg |= 1; + +out: + return page; +} + +void __free_pages_reserve(struct page *page, int order, + struct mem_reserve *res, int emerg) +{ + __free_pages(page, order); + mem_reserve_pages_charge(res, -(1 << order), 0); + wake_up_all(&res->waitqueue); +}