Commit bfbd4b6c authored by Vikram Narayanan's avatar Vikram Narayanan
Browse files

mm/privpool: Integrate the multipage hack



We do multiple buddy allocations to see if the returned regions are contiguous
and use those one single block.  Make changes in the init routine for
accommodating it.
Signed-off-by: Vikram Narayanan's avatarVikram Narayanan <vikram186@gmail.com>
parent 306f3e4c
......@@ -40,24 +40,32 @@ struct atom {
typedef struct {
struct object __percpu **head;
struct object __percpu **marker;
#ifdef PBUF
char __percpu **buf;
char __percpu **bufend;
#endif
int __percpu *cached;
unsigned int obj_size;
unsigned int total_pages;
unsigned int num_objs_percpu;
unsigned int total_objs;
unsigned int num_cpus;
void *pool;
void *gpool;
struct atom stack;
unsigned int pool_order;
spinlock_t pool_spin_lock;
bool dump_once;
} priv_pool_t;
void *priv_alloc(pool_type_t type);
void priv_free(void *p, pool_type_t type);
priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs, unsigned int obj_size);
//priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs, unsigned int obj_size);
priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
size_t pool_size,
unsigned int obj_size);
void priv_pool_destroy(priv_pool_t *p);
#endif /* PRIV_MEMPOOL_H */
......@@ -6,12 +6,24 @@
#include <linux/delay.h>
#include <linux/mm.h>
#include <linux/percpu.h>
#include <linux/cma.h>
#undef pr_fmt
#define pr_fmt(fmt) "%s:%d : " fmt, __func__, smp_processor_id()
//#define CMA_ALLOC
static priv_pool_t pool_array[POOL_MAX];
static unsigned long pool_cma_size = 1024 * 1024 * 50;
static struct cma *pool_cma;
static int __init early_parse_pool_cma(char *p)
{
pool_cma_size = ALIGN(memparse(p, NULL), PAGE_SIZE);
return 0;
}
early_param("pool_cma", early_parse_pool_cma);
static priv_pool_t *get_pool(pool_type_t type)
{
......@@ -22,13 +34,12 @@ static priv_pool_t *get_pool(pool_type_t type)
return &pool_array[type];
}
#define CACHE_SIZE 0x20
#define CACHE_SIZE 0x80
void construct_global_pool(priv_pool_t *p)
{
unsigned int list_sz = p->num_objs_percpu;
unsigned int obj_size = p->obj_size;
unsigned int gpool_objs = list_sz * num_possible_cpus();
unsigned int gpool_objs = p->total_objs;
char *gpool = p->gpool;
int i, b;
......@@ -42,15 +53,15 @@ void construct_global_pool(priv_pool_t *p)
*/
bundles = gpool_objs / CACHE_SIZE;
printk("%s, gpool_objs %d | list_sz %u | bundles %u\n",
__func__, gpool_objs, list_sz, bundles);
printk("%s, gpool_objs %d | bundles %u\n",
__func__, gpool_objs, bundles);
for (b = 0; b < bundles; b++) {
printk("bundle ===> %d\n", b);
//printk("bundle ===> %d\n", b);
for (i = 0; i < CACHE_SIZE; i++) {
objs = (struct object*)((char*)bpool + (i * obj_size));
objs->next = (struct object*)((char*)bpool + (i + 1) * obj_size);
printk("\tobj %p | obj->next %p\n", objs, objs->next);
// printk("\tobj %p | obj->next %p\n", objs, objs->next);
}
/* break the last object's chain */
objs->next = NULL;
......@@ -83,6 +94,25 @@ void construct_global_pool(priv_pool_t *p)
p->stack.head, p->stack.head->list, p->stack.head->next);
}
void priv_pool_dumpstats(priv_pool_t *p)
{
int cpu;
printk("======================== priv pool stats ======================\n");
for_each_online_cpu(cpu) {
printk("[cpu:%d] pool %p | p->head %p | p->marker %p | p->cached %d\n",
cpu, p,
*per_cpu_ptr(p->head, cpu),
*per_cpu_ptr(p->marker, cpu),
*per_cpu_ptr(p->cached, cpu));
}
printk("global pool stack head %p | head->list %p | head->next %p\n",
p->stack.head, p->stack.head->list, p->stack.head->next);
p->dump_once = true;
printk("===============================================================\n");
}
void priv_pool_destroy(priv_pool_t *p)
{
if (!p)
......@@ -91,12 +121,13 @@ void priv_pool_destroy(priv_pool_t *p)
if (p->pool)
free_pages((unsigned long)p->pool, p->pool_order);
#ifdef PBUF
if (p->buf)
free_percpu(p->buf);
if (p->bufend)
free_percpu(p->bufend);
#endif
if (p->head)
free_percpu(p->head);
......@@ -108,12 +139,19 @@ void priv_pool_destroy(priv_pool_t *p)
}
EXPORT_SYMBOL(priv_pool_destroy);
#if 0
priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
unsigned int obj_size)
{
priv_pool_t *p;
unsigned int num_pages, num_objs_percpu;
char *pool, *pcpu_pool, *global_pool;
char *pool, *global_pool;
#ifdef PBUF
char *pcpu_pool;
#endif
#ifdef CMA_ALLOC
struct page *page;
#endif
unsigned int total_pages, num_cpus;
int cpu;
......@@ -130,8 +168,10 @@ priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
memset(p, 0, sizeof(priv_pool_t));
#ifdef PBUF
p->buf = alloc_percpu(char *);
p->bufend = alloc_percpu(char *);
#endif
p->head = alloc_percpu(struct object *);
p->marker = alloc_percpu(struct object *);
p->cached = alloc_percpu(int);
......@@ -151,21 +191,34 @@ priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
* one set is for the percpu buf, the remaining pages
* would be given to the global buffer
*/
p->total_pages = total_pages = num_pages * (num_cpus + num_possible_cpus() * 2);
//p->total_pages = total_pages = num_pages * (num_cpus + num_online_cpus() * 2);
p->total_pages = total_pages = num_pages * num_cpus;
#ifdef PBUF
printk("num objs %d | num_cpus %d | num_pages %d | num_objs_percpu %d "
"| total_pages %d | page order %d\npcpu_pool %p | global_pool %p\n",
num_objs, num_online_cpus(), num_pages, num_objs_percpu,
total_pages, get_order(total_pages * PAGE_SIZE), pcpu_pool,
global_pool);
#else
printk("num objs %d | num_cpus %d | num_pages %d | num_objs_percpu %d "
"| total_pages %d | page order %d\n | global_pool %p\n",
num_objs, num_online_cpus(), num_pages, num_objs_percpu,
total_pages, get_order(total_pages * PAGE_SIZE),
global_pool);
#endif
p->pool_order = min_t(int, MAX_ORDER,
get_order(total_pages * PAGE_SIZE));
#ifdef CMA_ALLOC
//page = cma_alloc(pool_cma, total_pages * 16, 0, GFP_KERNEL);
page = cma_alloc(pool_cma, pool_cma_size >> PAGE_SHIFT, 0);
pool = p->pool = page_address(page);
p->total_objs = pool_cma_size / p->obj_size;
#else
/* alloc total_size pages */
pool = p->pool = (char*) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
p->pool_order);
#endif
if (!pool) {
printk("No memory %p\n", pool);
return NULL;
......@@ -173,23 +226,95 @@ priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
printk("Memory %p | size %lu\n", pool, total_pages * PAGE_SIZE);
}
/* split the total pages between pcpu pool and the global pool */
#ifdef PBUF
pcpu_pool = pool;
p->gpool = global_pool =
pool + (num_cpus * num_pages * PAGE_SIZE);
#else
p->gpool = global_pool = pool;
#endif
/* update percpu vars */
for_each_online_cpu(cpu) {
*per_cpu_ptr(p->marker, cpu) =
*per_cpu_ptr(p->head, cpu) = (struct object*) NULL;
#ifdef PBUF
*per_cpu_ptr(p->buf, cpu) =
pcpu_pool + (cpu * num_pages * PAGE_SIZE);
*per_cpu_ptr(p->bufend, cpu) =
pcpu_pool + ((cpu + 1) * num_pages * PAGE_SIZE) - 1;
*per_cpu_ptr(p->cached, cpu) = p->num_objs_percpu;
*per_cpu_ptr(p->cached, cpu) = CACHE_SIZE;
printk("cpu %d | buf %p | bufend %p\n",
cpu, *per_cpu_ptr(p->buf, cpu),
*per_cpu_ptr(p->bufend, cpu));
#endif
}
construct_global_pool(p);
spin_lock_init(&p->pool_spin_lock);
return p;
}
EXPORT_SYMBOL(priv_pool_init);
#endif
priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
size_t pool_size,
unsigned int obj_size)
{
priv_pool_t *p;
char *pool, *global_pool;
int cpu;
if (!obj_size) {
printk("%s, invalid objsize (%u) requested\n",
__func__, obj_size);
return NULL;
}
p = get_pool(type);
if (!p)
return NULL;
memset(p, 0, sizeof(priv_pool_t));
p->head = alloc_percpu(struct object *);
p->marker = alloc_percpu(struct object *);
p->cached = alloc_percpu(int);
/* align obj_size to 32 bit boundary */
p->obj_size = obj_size = ALIGN(obj_size, 32);
p->num_cpus = num_online_cpus() * 2;
p->pool_order = 10;
if (!pool_base) {
/* alloc total_size pages */
pool = p->pool = (char*) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
p->pool_order);
p->total_objs = ((1 << p->pool_order) * PAGE_SIZE)
/ p->obj_size;
printk("Memory %p | size %lx\n", pool, (1 << p->pool_order) * PAGE_SIZE);
} else {
pool = p->pool = pool_base;
p->total_objs = pool_size / p->obj_size;
printk("Memory %p | size %lx\n", pool, pool_size);
p->total_pages = pool_size >> PAGE_SHIFT;
}
if (!pool) {
printk("No memory %p\n", pool);
return NULL;
}
/* split the total pages between pcpu pool and the global pool */
p->gpool = global_pool = pool;
/* update percpu vars */
for_each_online_cpu(cpu) {
*per_cpu_ptr(p->marker, cpu) =
*per_cpu_ptr(p->head, cpu) = (struct object*) NULL;
}
construct_global_pool(p);
......@@ -201,8 +326,10 @@ EXPORT_SYMBOL(priv_pool_init);
void *priv_alloc(pool_type_t type)
{
void *m = NULL;
#ifdef PBUF
char *pbuf;
char *pbufend;
#endif
struct object *head;
priv_pool_t *p = get_pool(type);
......@@ -226,6 +353,7 @@ void *priv_alloc(pool_type_t type)
this_cpu_write(*(p->cached), 0);
}
#ifdef PBUF
pbuf = (char*)*this_cpu_ptr(p->buf);
pbufend = (char*)*this_cpu_ptr(p->bufend);
......@@ -242,13 +370,15 @@ void *priv_alloc(pool_type_t type)
/* got a chunk */
goto out;
}
#endif
/* no cached or private pool chunk. Try our luck in the
* global pool. First, check if global pool has any chunk
* remaining
*/
if (!p->stack.head) {
pr_err("Out of memory! This must be crazy\n");
WARN_ONCE(!p->stack.head, "Out of memory! This must be crazy\n");
if (!p->dump_once)
priv_pool_dumpstats(p);
goto out;
}
{
......@@ -274,15 +404,22 @@ void *priv_alloc(pool_type_t type)
p->stack.version = new.version;
*this_cpu_ptr(p->head) = snapshot.head->list;
#ifdef PBUF
this_cpu_write(*(p->cached), CACHE_SIZE);
#else
this_cpu_write(*(p->cached), CACHE_SIZE - 1);
#endif
/* unlock global pool */
spin_unlock(&p->pool_spin_lock)
#else
if (cmpxchg_double(&p->stack.head, &p->stack.version, snapshot.head, snapshot.version,
new.head, new.version)) {
*this_cpu_ptr(p->head) = snapshot.head->list;
#ifdef PBUF
this_cpu_write(*(p->cached), CACHE_SIZE);
#else
this_cpu_write(*(p->cached), CACHE_SIZE - 1);
#endif
m = (struct object*) snapshot.head;
pr_debug("from gpool ohead: %p/%ld, nhead: %p/%ld\n",
......@@ -295,7 +432,6 @@ void *priv_alloc(pool_type_t type)
} // for
#endif
}
out:
/* enable preemption */
preempt_enable();
......@@ -327,9 +463,19 @@ void priv_free(void *addr, pool_type_t type)
pr_debug("chaining %p to head %p\n", p, head);
#ifdef PBUF
if (this_cpu_inc_return(*pool->cached) == CACHE_SIZE) {
#else
if (this_cpu_inc_return(*pool->cached) == CACHE_SIZE + 1) {
#endif
*this_cpu_ptr(pool->marker) = *this_cpu_ptr(pool->head);
pr_debug("set marker @ %p\n", *this_cpu_ptr(pool->marker));
if ((void*)(*this_cpu_ptr(pool->marker))->next < (void*)pool->pool ||
(void*)(*this_cpu_ptr(pool->marker))->next > (void*)(pool->pool + (pool->total_pages << PAGE_SHIFT))) {
printk("marker->next is corrupted!! marker %p | marker->next %p\n",
*this_cpu_ptr(pool->marker), (*this_cpu_ptr(pool->marker))->next);
dump_stack();
}
} else if (*this_cpu_ptr(pool->cached) == (CACHE_SIZE << 1)) {
struct bundle *donation = (struct bundle *)*this_cpu_ptr(pool->head);
struct atom snapshot, new;
......@@ -341,10 +487,21 @@ void priv_free(void *addr, pool_type_t type)
new.head = donation;
donation->list = ((struct object*)*this_cpu_ptr(pool->head))->next;
if ((void*)(*this_cpu_ptr(pool->marker))->next < (void*)pool->pool ||
(void*)(*this_cpu_ptr(pool->marker))->next > (void*)(pool->pool + (pool->total_pages << PAGE_SHIFT))) {
printk("update pool->head with corrupted marker %p | marker->next %p\n",
*this_cpu_ptr(pool->marker), (*this_cpu_ptr(pool->marker))->next);
dump_stack();
}
//BUG_ON((*this_cpu_ptr(pool->marker))->next < pool->pool || (*this_cpu_ptr(pool->marker))->next > (pool->pool + pool->total_pages))
*this_cpu_ptr(pool->head) = (*this_cpu_ptr(pool->marker))->next;
(*this_cpu_ptr(pool->marker))->next = NULL;
#ifdef PBUF
*this_cpu_ptr(pool->cached) = CACHE_SIZE - 1;
#else
*this_cpu_ptr(pool->cached) = CACHE_SIZE;
#endif
#ifndef SPINLOCK
do {
#endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment