...
 
Commits (2)
......@@ -62,6 +62,7 @@ typedef struct {
unsigned int pool_order;
spinlock_t pool_spin_lock;
bool dump_once;
struct dentry *stats;
} priv_pool_t;
......@@ -74,4 +75,6 @@ priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
unsigned int obj_size);
void priv_pool_destroy(priv_pool_t *p);
void priv_pool_dumpstats(priv_pool_t *p);
#endif /* PRIV_MEMPOOL_H */
......@@ -7,22 +7,16 @@
#include <linux/mm.h>
#include <linux/percpu.h>
#include <linux/cma.h>
#include <linux/debugfs.h>
#undef pr_fmt
#define pr_fmt(fmt) "%s:%d : " fmt, __func__, smp_processor_id()
//#define CMA_ALLOC
static priv_pool_t pool_array[POOL_MAX];
static unsigned long pool_cma_size = 1024 * 1024 * 50;
static struct cma *pool_cma;
#define SPINLOCK
static int __init early_parse_pool_cma(char *p)
{
pool_cma_size = ALIGN(memparse(p, NULL), PAGE_SIZE);
return 0;
}
early_param("pool_cma", early_parse_pool_cma);
static priv_pool_t pool_array[POOL_MAX];
struct dentry *mempool_debugfs_dir;
struct dentry *g_stats_file;
static priv_pool_t *get_pool(pool_type_t type)
{
......@@ -34,7 +28,7 @@ static priv_pool_t *get_pool(pool_type_t type)
return &pool_array[type];
}
#define CACHE_SIZE 0x80
#define CACHE_SIZE 0x100
void construct_global_pool(priv_pool_t *p)
{
......@@ -108,18 +102,27 @@ void priv_pool_dumpstats(priv_pool_t *p)
}
printk("global pool stack head %p | head->list %p | head->next %p\n",
p->stack.head, p->stack.head->list, p->stack.head->next);
p->stack.head, p->stack.head ? p->stack.head->list : NULL,
p->stack.head ? p->stack.head->next : NULL);
p->dump_once = true;
printk("===============================================================\n");
}
EXPORT_SYMBOL(priv_pool_dumpstats);
void priv_pool_destroy(priv_pool_t *p)
{
if (!p)
return;
/*
* XXX: Let the creator free it Easy to handle between kmalloc/vmalloc
* allocations
*/
#if 0
if (p->pool)
free_pages((unsigned long)p->pool, p->pool_order);
#endif
#ifdef PBUF
if (p->buf)
......@@ -288,9 +291,9 @@ priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
p->num_cpus = num_online_cpus() * 2;
p->pool_order = 10;
if (!pool_base) {
p->pool_order = 10;
/* alloc total_size pages */
pool = p->pool = (char*) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
p->pool_order);
......@@ -331,6 +334,9 @@ void *priv_alloc(pool_type_t type)
char *pbufend;
#endif
struct object *head;
bool retry = true;
unsigned long flags;
priv_pool_t *p = get_pool(type);
if (!p)
......@@ -339,6 +345,13 @@ void *priv_alloc(pool_type_t type)
/* disable preempt until we manipulate all percpu pointers */
preempt_disable();
#ifdef SPINLOCK
/* lock global pool */
spin_lock_irqsave(&p->pool_spin_lock, flags);
#endif
retry:
head = (struct object*) *this_cpu_ptr(p->head);
/* if head is not null */
......@@ -379,6 +392,12 @@ void *priv_alloc(pool_type_t type)
WARN_ONCE(!p->stack.head, "Out of memory! This must be crazy\n");
if (!p->dump_once)
priv_pool_dumpstats(p);
if (retry) {
retry = false;
goto retry;
}
goto out;
}
{
......@@ -387,10 +406,6 @@ void *priv_alloc(pool_type_t type)
#endif
struct atom snapshot, new;
#ifdef SPINLOCK
/* lock global pool */
spin_lock(&p->pool_spin_lock);
#endif
// this is old
snapshot = p->stack;
......@@ -409,8 +424,7 @@ void *priv_alloc(pool_type_t type)
#else
this_cpu_write(*(p->cached), CACHE_SIZE - 1);
#endif
/* unlock global pool */
spin_unlock(&p->pool_spin_lock)
#else
if (cmpxchg_double(&p->stack.head, &p->stack.version, snapshot.head, snapshot.version,
new.head, new.version)) {
......@@ -433,6 +447,11 @@ void *priv_alloc(pool_type_t type)
#endif
}
out:
#ifdef SPINLOCK
/* unlock global pool */
spin_unlock_irqrestore(&p->pool_spin_lock, flags);
#endif
/* enable preemption */
preempt_enable();
return m;
......@@ -444,6 +463,8 @@ void priv_free(void *addr, pool_type_t type)
{
struct object *p, *head;
priv_pool_t *pool;
unsigned long flags;
if (!addr)
return;
......@@ -457,6 +478,11 @@ void priv_free(void *addr, pool_type_t type)
/* disable preempt until we manipulate all percpu pointers */
preempt_disable();
#ifdef SPINLOCK
/* lock global pool */
spin_lock_irqsave(&pool->pool_spin_lock, flags);
#endif
head = (struct object*)*this_cpu_ptr(pool->head);
p->next = (struct object*)head;
*this_cpu_ptr(pool->head) = p;
......@@ -480,10 +506,6 @@ void priv_free(void *addr, pool_type_t type)
struct bundle *donation = (struct bundle *)*this_cpu_ptr(pool->head);
struct atom snapshot, new;
#ifdef SPINLOCK
/* lock global pool */
spin_lock(&pool->pool_spin_lock);
#endif
new.head = donation;
donation->list = ((struct object*)*this_cpu_ptr(pool->head))->next;
......@@ -513,9 +535,6 @@ void priv_free(void *addr, pool_type_t type)
#ifdef SPINLOCK
pool->stack.head = new.head;
pool->stack.version = new.version;
/* unlock global pool */
spin_unlock(&pool->pool_spin_lock);
#else
} while (!cmpxchg_double(&pool->stack.head, &pool->stack.version, snapshot.head, snapshot.version,
new.head, new.version));
......@@ -527,19 +546,66 @@ void priv_free(void *addr, pool_type_t type)
new.head, new.version);
}
#ifdef SPINLOCK
/* unlock global pool */
spin_unlock_irqrestore(&pool->pool_spin_lock, flags);
#endif
/* enable preemption */
preempt_enable();
}
EXPORT_SYMBOL(priv_free);
static int stats_print(struct seq_file *s, void *unused)
{
int i;
seq_printf(s, "%8s %8s %5s %5s %8s\n", "pool", "poolbase", "poolpages",
"objsize", "totalobjs");
for (i = 0; i < POOL_MAX; i++) {
priv_pool_t *p = &pool_array[i];
switch (i) {
case SKB_DATA_POOL:
seq_printf(s, "%16s ", "skb_data_pool");
break;
case SKB_FRAG_POOL:
seq_printf(s, "%16s ", "skb_frag_pool");
break;
case SKB_CONTAINER_POOL:
seq_printf(s, "%16s ", "skb_container_pool");
break;
}
seq_printf(s, "%016lx %05u %05u %05u\n", (unsigned long)
p->pool, p->total_pages, p->obj_size,
p->total_objs);
}
return 0;
}
static int stats_open(struct inode *inode, struct file *file)
{
return single_open(file, stats_print, inode->i_private);
}
static const struct file_operations debugfs_ops = {
.open = stats_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
int priv_init(void)
{
mempool_debugfs_dir = debugfs_create_dir("priv_mempool", 0);
g_stats_file = debugfs_create_file("stats", 0400, mempool_debugfs_dir,
NULL, &debugfs_ops);
return 0;
}
module_init(priv_init);
void priv_exit(void)
{
debugfs_remove(g_stats_file);
debugfs_remove(mempool_debugfs_dir);
return;
}
module_exit(priv_exit);
......