mm/privpool: Integrate the multipage hack

We do multiple buddy allocations to see if the returned regions are contiguous
and use those one single block.  Make changes in the init routine for
accommodating it.

Signed-off-by: Vikram Narayanan <vikram186@gmail.com>

include/linux/priv_mempool.h

@@ -40,24 +40,32 @@ struct atom {
 typedef struct {
 	struct object __percpu **head;
 	struct object __percpu **marker;
+#ifdef PBUF
 	char __percpu **buf;
 	char __percpu **bufend;
+#endif
 	int __percpu *cached;
 	unsigned int obj_size;
 	unsigned int total_pages;
 	unsigned int num_objs_percpu;
+	unsigned int total_objs;
 	unsigned int num_cpus;
 	void *pool;
 	void *gpool;
 	struct atom stack;
 	unsigned int pool_order;
 	spinlock_t pool_spin_lock;
+	bool dump_once;
 } priv_pool_t;
 
 
 void *priv_alloc(pool_type_t type);
 void priv_free(void *p, pool_type_t type);
-priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs, unsigned int obj_size);
+//priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs, unsigned int obj_size);
+
+priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
+			size_t pool_size,
+			unsigned int obj_size);
 void priv_pool_destroy(priv_pool_t *p);
 
 #endif /* PRIV_MEMPOOL_H */

mm/priv_mempool.c

@@ -6,12 +6,24 @@
 #include <linux/delay.h>
 #include <linux/mm.h>
 #include <linux/percpu.h>
+#include <linux/cma.h>
 
 #undef pr_fmt
 #define pr_fmt(fmt)	"%s:%d : " fmt, __func__, smp_processor_id()
 
+//#define CMA_ALLOC
 static priv_pool_t pool_array[POOL_MAX];
 
+static unsigned long pool_cma_size = 1024 * 1024 * 50;
+static struct cma *pool_cma;
+
+static int __init early_parse_pool_cma(char *p)
+{
+	pool_cma_size = ALIGN(memparse(p, NULL), PAGE_SIZE);
+	return 0;
+}
+early_param("pool_cma", early_parse_pool_cma);
+
 static priv_pool_t *get_pool(pool_type_t type)
 {
 
@@ -22,13 +34,12 @@ static priv_pool_t *get_pool(pool_type_t type)
 	return &pool_array[type];
 }
 
-#define CACHE_SIZE	0x20
+#define CACHE_SIZE	0x80
 
 void construct_global_pool(priv_pool_t *p)
 {
-	unsigned int list_sz = p->num_objs_percpu;
 	unsigned int obj_size = p->obj_size;
-	unsigned int gpool_objs = list_sz * num_possible_cpus();
+	unsigned int gpool_objs = p->total_objs;
 
 	char *gpool = p->gpool;
 	int i, b;
@@ -42,15 +53,15 @@ void construct_global_pool(priv_pool_t *p)
 	 */
 	bundles = gpool_objs / CACHE_SIZE;
 
-	printk("%s, gpool_objs %d | list_sz %u | bundles %u\n",
-		__func__, gpool_objs, list_sz, bundles);
+	printk("%s, gpool_objs %d | bundles %u\n",
+		__func__, gpool_objs, bundles);
 
 	for (b = 0; b < bundles; b++) {
-		printk("bundle ===> %d\n", b);
+		//printk("bundle ===> %d\n", b);
 		for (i = 0; i < CACHE_SIZE; i++) {
 			objs = (struct object*)((char*)bpool + (i * obj_size));
 			objs->next = (struct object*)((char*)bpool + (i + 1) * obj_size);
-			printk("\tobj %p | obj->next %p\n", objs, objs->next);
+	//		printk("\tobj %p | obj->next %p\n", objs, objs->next);
 		}
 		/* break the last object's chain */
 		objs->next = NULL;
@@ -83,6 +94,25 @@ void construct_global_pool(priv_pool_t *p)
 		p->stack.head, p->stack.head->list, p->stack.head->next);
 }
 
+void priv_pool_dumpstats(priv_pool_t *p)
+{
+	int cpu;
+	printk("======================== priv pool stats ======================\n");
+
+	for_each_online_cpu(cpu) {
+	printk("[cpu:%d] pool %p | p->head %p | p->marker %p | p->cached %d\n",
+			cpu, p,
+			*per_cpu_ptr(p->head, cpu),
+			*per_cpu_ptr(p->marker, cpu),
+			*per_cpu_ptr(p->cached, cpu));
+	}
+
+	printk("global pool stack head %p | head->list %p | head->next %p\n",
+		p->stack.head, p->stack.head->list, p->stack.head->next);
+	p->dump_once = true;
+	printk("===============================================================\n");
+}
+
 void priv_pool_destroy(priv_pool_t *p)
 {
 	if (!p)
@@ -91,12 +121,13 @@ void priv_pool_destroy(priv_pool_t *p)
 	if (p->pool)
 		free_pages((unsigned long)p->pool, p->pool_order);
 
+#ifdef PBUF
 	if (p->buf)
 		free_percpu(p->buf);
 
 	if (p->bufend)
 		free_percpu(p->bufend);
-
+#endif
 	if (p->head)
 		free_percpu(p->head);
 
@@ -108,12 +139,19 @@ void priv_pool_destroy(priv_pool_t *p)
 }
 EXPORT_SYMBOL(priv_pool_destroy);
 
+#if 0
 priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
             unsigned int obj_size)
 {
 	priv_pool_t *p;
 	unsigned int num_pages, num_objs_percpu;
-	char *pool, *pcpu_pool, *global_pool;
+	char *pool, *global_pool;
+#ifdef PBUF
+	char *pcpu_pool;
+#endif
+#ifdef CMA_ALLOC
+	struct page *page;
+#endif
 	unsigned int total_pages, num_cpus;
 	int cpu;
 
@@ -130,8 +168,10 @@ priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
 
 	memset(p, 0, sizeof(priv_pool_t));
 
+#ifdef PBUF
 	p->buf = alloc_percpu(char *);
 	p->bufend = alloc_percpu(char *);
+#endif
 	p->head = alloc_percpu(struct object *);
 	p->marker = alloc_percpu(struct object *);
 	p->cached = alloc_percpu(int);
@@ -151,21 +191,34 @@ priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
 	 * one set is for the percpu buf, the remaining pages
 	 * would be given to the global buffer
 	 */
-	p->total_pages = total_pages = num_pages * (num_cpus + num_possible_cpus() * 2);
-
+	//p->total_pages = total_pages = num_pages * (num_cpus + num_online_cpus() * 2);
+	p->total_pages = total_pages = num_pages * num_cpus;
+#ifdef PBUF
 	printk("num objs %d | num_cpus %d | num_pages %d | num_objs_percpu %d "
 		"| total_pages %d | page order %d\npcpu_pool %p | global_pool %p\n",
 		num_objs, num_online_cpus(), num_pages, num_objs_percpu,
 		total_pages, get_order(total_pages * PAGE_SIZE), pcpu_pool,
 		global_pool);
-
+#else
+	printk("num objs %d | num_cpus %d | num_pages %d | num_objs_percpu %d "
+		"| total_pages %d | page order %d\n | global_pool %p\n",
+		num_objs, num_online_cpus(), num_pages, num_objs_percpu,
+		total_pages, get_order(total_pages * PAGE_SIZE),
+		global_pool);
+#endif
 	p->pool_order = min_t(int, MAX_ORDER,
 	                        get_order(total_pages * PAGE_SIZE));
 
+#ifdef CMA_ALLOC
+	//page = cma_alloc(pool_cma, total_pages * 16, 0, GFP_KERNEL);
+	page = cma_alloc(pool_cma, pool_cma_size >> PAGE_SHIFT, 0);
+	pool = p->pool = page_address(page);
+	p->total_objs = pool_cma_size / p->obj_size;
+#else
 	/* alloc total_size pages */
 	pool = p->pool = (char*) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
 	                            p->pool_order);
-
+#endif
 	if (!pool) {
 		printk("No memory %p\n", pool);
 		return NULL;
@@ -173,23 +226,95 @@ priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
 		printk("Memory %p | size %lu\n", pool, total_pages * PAGE_SIZE);
 	}
 	/* split the total pages between pcpu pool and the global pool */
+#ifdef PBUF
 	pcpu_pool = pool;
 	p->gpool = global_pool =
 	                pool + (num_cpus * num_pages * PAGE_SIZE);
+#else
+	p->gpool = global_pool = pool;
+#endif
 
 	/* update percpu vars */
 	for_each_online_cpu(cpu) {
 		*per_cpu_ptr(p->marker, cpu) =
 		        *per_cpu_ptr(p->head, cpu) = (struct object*) NULL;
+#ifdef PBUF
 		*per_cpu_ptr(p->buf, cpu) =
 		        pcpu_pool + (cpu * num_pages * PAGE_SIZE);
 		*per_cpu_ptr(p->bufend, cpu) =
 		        pcpu_pool + ((cpu + 1) * num_pages * PAGE_SIZE) - 1;
 		*per_cpu_ptr(p->cached, cpu) = p->num_objs_percpu;
-
+		*per_cpu_ptr(p->cached, cpu) = CACHE_SIZE;
 		printk("cpu %d | buf %p | bufend %p\n",
 				cpu, *per_cpu_ptr(p->buf, cpu),
 				*per_cpu_ptr(p->bufend, cpu));
+#endif
+	}
+
+	construct_global_pool(p);
+	spin_lock_init(&p->pool_spin_lock);
+	return p;
+}
+EXPORT_SYMBOL(priv_pool_init);
+#endif
+
+priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
+			size_t pool_size,
+			unsigned int obj_size)
+{
+	priv_pool_t *p;
+	char *pool, *global_pool;
+	int cpu;
+
+	if (!obj_size) {
+		printk("%s, invalid objsize (%u) requested\n",
+		            __func__, obj_size);
+		return NULL;
+	}
+
+	p = get_pool(type);
+
+	if (!p)
+		return NULL;
+
+	memset(p, 0, sizeof(priv_pool_t));
+
+	p->head = alloc_percpu(struct object *);
+	p->marker = alloc_percpu(struct object *);
+	p->cached = alloc_percpu(int);
+
+	/* align obj_size to 32 bit boundary */
+	p->obj_size = obj_size = ALIGN(obj_size, 32);
+
+	p->num_cpus = num_online_cpus() * 2;
+
+	p->pool_order = 10;
+
+	if (!pool_base) {
+		/* alloc total_size pages */
+		pool = p->pool = (char*) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
+	                            p->pool_order);
+		p->total_objs = ((1 << p->pool_order) * PAGE_SIZE)
+						/ p->obj_size;
+		printk("Memory %p | size %lx\n", pool, (1 << p->pool_order) * PAGE_SIZE);
+	} else {
+		pool = p->pool = pool_base;
+		p->total_objs = pool_size / p->obj_size;
+		printk("Memory %p | size %lx\n", pool, pool_size);
+		p->total_pages = pool_size >> PAGE_SHIFT;
+	}
+
+	if (!pool) {
+		printk("No memory %p\n", pool);
+		return NULL;
+	}
+	/* split the total pages between pcpu pool and the global pool */
+	p->gpool = global_pool = pool;
+
+	/* update percpu vars */
+	for_each_online_cpu(cpu) {
+		*per_cpu_ptr(p->marker, cpu) =
+		        *per_cpu_ptr(p->head, cpu) = (struct object*) NULL;
 	}
 
 	construct_global_pool(p);
@@ -201,8 +326,10 @@ EXPORT_SYMBOL(priv_pool_init);
 void *priv_alloc(pool_type_t type)
 {
 	void *m = NULL;
+#ifdef PBUF
 	char *pbuf;
 	char *pbufend;
+#endif
 	struct object *head;
 	priv_pool_t *p = get_pool(type);
 
@@ -226,6 +353,7 @@ void *priv_alloc(pool_type_t type)
 		this_cpu_write(*(p->cached), 0);
 	}
 
+#ifdef PBUF
 	pbuf = (char*)*this_cpu_ptr(p->buf);
 	pbufend = (char*)*this_cpu_ptr(p->bufend);
 
@@ -242,13 +370,15 @@ void *priv_alloc(pool_type_t type)
 		/* got a chunk */
 		goto out;
 	}
-
+#endif
 	/* no cached or private pool chunk. Try our luck in the
 	* global pool. First, check if global pool has any chunk
 	* remaining
 	*/
 	if (!p->stack.head) {
-		pr_err("Out of memory! This must be crazy\n");
+		WARN_ONCE(!p->stack.head, "Out of memory! This must be crazy\n");
+		if (!p->dump_once)
+			priv_pool_dumpstats(p);
 		goto out;
 	}
 	{
@@ -274,15 +404,22 @@ void *priv_alloc(pool_type_t type)
 		p->stack.version = new.version;
 
 		*this_cpu_ptr(p->head) = snapshot.head->list;
+#ifdef PBUF
 		this_cpu_write(*(p->cached), CACHE_SIZE);
-
+#else
+		this_cpu_write(*(p->cached), CACHE_SIZE - 1);
+#endif
 		/* unlock global pool */
 		spin_unlock(&p->pool_spin_lock)
 #else
 		if (cmpxchg_double(&p->stack.head, &p->stack.version, snapshot.head, snapshot.version,
 					new.head, new.version)) {
 			*this_cpu_ptr(p->head) = snapshot.head->list;
+#ifdef PBUF
 			this_cpu_write(*(p->cached), CACHE_SIZE);
+#else
+			this_cpu_write(*(p->cached), CACHE_SIZE - 1);
+#endif
 			m = (struct object*) snapshot.head;
 
 			pr_debug("from gpool ohead: %p/%ld, nhead: %p/%ld\n",
@@ -295,7 +432,6 @@ void *priv_alloc(pool_type_t type)
 		} // for
 #endif
 	}
-
 out:
 	/* enable preemption */
 	preempt_enable();
@@ -327,9 +463,19 @@ void priv_free(void *addr, pool_type_t type)
 
 	pr_debug("chaining %p to head %p\n", p, head);
 
+#ifdef PBUF
 	if (this_cpu_inc_return(*pool->cached) == CACHE_SIZE) {
+#else
+	if (this_cpu_inc_return(*pool->cached) == CACHE_SIZE + 1) {
+#endif
 		*this_cpu_ptr(pool->marker) = *this_cpu_ptr(pool->head);
 		pr_debug("set marker @ %p\n", *this_cpu_ptr(pool->marker));
+		if ((void*)(*this_cpu_ptr(pool->marker))->next < (void*)pool->pool ||
+				(void*)(*this_cpu_ptr(pool->marker))->next > (void*)(pool->pool + (pool->total_pages << PAGE_SHIFT))) {
+			printk("marker->next is corrupted!! marker %p | marker->next %p\n",
+						*this_cpu_ptr(pool->marker), (*this_cpu_ptr(pool->marker))->next);
+			dump_stack();
+		}
 	} else if (*this_cpu_ptr(pool->cached) == (CACHE_SIZE << 1)) {
 		struct bundle *donation = (struct bundle *)*this_cpu_ptr(pool->head);
 		struct atom snapshot, new;
@@ -341,10 +487,21 @@ void priv_free(void *addr, pool_type_t type)
 
 		new.head = donation;
 		donation->list = ((struct object*)*this_cpu_ptr(pool->head))->next;
+
+		if ((void*)(*this_cpu_ptr(pool->marker))->next < (void*)pool->pool ||
+				(void*)(*this_cpu_ptr(pool->marker))->next > (void*)(pool->pool + (pool->total_pages << PAGE_SHIFT))) {
+			printk("update pool->head with corrupted marker %p | marker->next %p\n",
+						*this_cpu_ptr(pool->marker), (*this_cpu_ptr(pool->marker))->next);
+			dump_stack();
+		}
+		//BUG_ON((*this_cpu_ptr(pool->marker))->next < pool->pool || (*this_cpu_ptr(pool->marker))->next > (pool->pool + pool->total_pages))
 		*this_cpu_ptr(pool->head) = (*this_cpu_ptr(pool->marker))->next;
 		(*this_cpu_ptr(pool->marker))->next = NULL;
+#ifdef PBUF
 		*this_cpu_ptr(pool->cached) = CACHE_SIZE - 1;
-
+#else
+		*this_cpu_ptr(pool->cached) = CACHE_SIZE;
+#endif
 #ifndef SPINLOCK
 		do {
 #endif