Commit f89bd6f8 authored by Thomas Graf's avatar Thomas Graf Committed by David S. Miller

rhashtable: Supports for nulls marker

In order to allow for wider usage of rhashtable, use a special nulls
marker to terminate each chain. The reason for not using the existing
nulls_list is that the prev pointer usage would not be valid as entries
can be linked in two different buckets at the same time.

The 4 nulls base bits can be set through the rhashtable_params structure
like this:

struct rhashtable_params params = {
        [...]
        .nulls_base = (1U << RHT_BASE_SHIFT),
};

This reduces the hash length from 32 bits to 27 bits.
Signed-off-by: default avatarThomas Graf <tgraf@suug.ch>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 97defe1e
...@@ -21,8 +21,9 @@ struct hlist_nulls_head { ...@@ -21,8 +21,9 @@ struct hlist_nulls_head {
struct hlist_nulls_node { struct hlist_nulls_node {
struct hlist_nulls_node *next, **pprev; struct hlist_nulls_node *next, **pprev;
}; };
#define NULLS_MARKER(value) (1UL | (((long)value) << 1))
#define INIT_HLIST_NULLS_HEAD(ptr, nulls) \ #define INIT_HLIST_NULLS_HEAD(ptr, nulls) \
((ptr)->first = (struct hlist_nulls_node *) (1UL | (((long)nulls) << 1))) ((ptr)->first = (struct hlist_nulls_node *) NULLS_MARKER(nulls))
#define hlist_nulls_entry(ptr, type, member) container_of(ptr,type,member) #define hlist_nulls_entry(ptr, type, member) container_of(ptr,type,member)
/** /**
......
...@@ -18,15 +18,32 @@ ...@@ -18,15 +18,32 @@
#ifndef _LINUX_RHASHTABLE_H #ifndef _LINUX_RHASHTABLE_H
#define _LINUX_RHASHTABLE_H #define _LINUX_RHASHTABLE_H
#include <linux/rculist.h> #include <linux/list_nulls.h>
#include <linux/workqueue.h> #include <linux/workqueue.h>
/*
* The end of the chain is marked with a special nulls marks which has
* the following format:
*
* +-------+-----------------------------------------------------+-+
* | Base | Hash |1|
* +-------+-----------------------------------------------------+-+
*
* Base (4 bits) : Reserved to distinguish between multiple tables.
* Specified via &struct rhashtable_params.nulls_base.
* Hash (27 bits): Full hash (unmasked) of first element added to bucket
* 1 (1 bit) : Nulls marker (always set)
*
* The remaining bits of the next pointer remain unused for now.
*/
#define RHT_BASE_BITS 4
#define RHT_HASH_BITS 27
#define RHT_BASE_SHIFT RHT_HASH_BITS
struct rhash_head { struct rhash_head {
struct rhash_head __rcu *next; struct rhash_head __rcu *next;
}; };
#define INIT_HASH_HEAD(ptr) ((ptr)->next = NULL)
/** /**
* struct bucket_table - Table of hash buckets * struct bucket_table - Table of hash buckets
* @size: Number of hash buckets * @size: Number of hash buckets
...@@ -55,6 +72,7 @@ struct rhashtable; ...@@ -55,6 +72,7 @@ struct rhashtable;
* @hash_rnd: Seed to use while hashing * @hash_rnd: Seed to use while hashing
* @max_shift: Maximum number of shifts while expanding * @max_shift: Maximum number of shifts while expanding
* @min_shift: Minimum number of shifts while shrinking * @min_shift: Minimum number of shifts while shrinking
* @nulls_base: Base value to generate nulls marker
* @locks_mul: Number of bucket locks to allocate per cpu (default: 128) * @locks_mul: Number of bucket locks to allocate per cpu (default: 128)
* @hashfn: Function to hash key * @hashfn: Function to hash key
* @obj_hashfn: Function to hash object * @obj_hashfn: Function to hash object
...@@ -69,6 +87,7 @@ struct rhashtable_params { ...@@ -69,6 +87,7 @@ struct rhashtable_params {
u32 hash_rnd; u32 hash_rnd;
size_t max_shift; size_t max_shift;
size_t min_shift; size_t min_shift;
u32 nulls_base;
size_t locks_mul; size_t locks_mul;
rht_hashfn_t hashfn; rht_hashfn_t hashfn;
rht_obj_hashfn_t obj_hashfn; rht_obj_hashfn_t obj_hashfn;
...@@ -100,6 +119,24 @@ struct rhashtable { ...@@ -100,6 +119,24 @@ struct rhashtable {
bool being_destroyed; bool being_destroyed;
}; };
static inline unsigned long rht_marker(const struct rhashtable *ht, u32 hash)
{
return NULLS_MARKER(ht->p.nulls_base + hash);
}
#define INIT_RHT_NULLS_HEAD(ptr, ht, hash) \
((ptr) = (typeof(ptr)) rht_marker(ht, hash))
static inline bool rht_is_a_nulls(const struct rhash_head *ptr)
{
return ((unsigned long) ptr & 1);
}
static inline unsigned long rht_get_nulls_value(const struct rhash_head *ptr)
{
return ((unsigned long) ptr) >> 1;
}
#ifdef CONFIG_PROVE_LOCKING #ifdef CONFIG_PROVE_LOCKING
int lockdep_rht_mutex_is_held(struct rhashtable *ht); int lockdep_rht_mutex_is_held(struct rhashtable *ht);
int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash); int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash);
...@@ -157,7 +194,7 @@ void rhashtable_destroy(struct rhashtable *ht); ...@@ -157,7 +194,7 @@ void rhashtable_destroy(struct rhashtable *ht);
*/ */
#define rht_for_each_continue(pos, head, tbl, hash) \ #define rht_for_each_continue(pos, head, tbl, hash) \
for (pos = rht_dereference_bucket(head, tbl, hash); \ for (pos = rht_dereference_bucket(head, tbl, hash); \
pos; \ !rht_is_a_nulls(pos); \
pos = rht_dereference_bucket((pos)->next, tbl, hash)) pos = rht_dereference_bucket((pos)->next, tbl, hash))
/** /**
...@@ -180,7 +217,7 @@ void rhashtable_destroy(struct rhashtable *ht); ...@@ -180,7 +217,7 @@ void rhashtable_destroy(struct rhashtable *ht);
*/ */
#define rht_for_each_entry_continue(tpos, pos, head, tbl, hash, member) \ #define rht_for_each_entry_continue(tpos, pos, head, tbl, hash, member) \
for (pos = rht_dereference_bucket(head, tbl, hash); \ for (pos = rht_dereference_bucket(head, tbl, hash); \
pos && rht_entry(tpos, pos, member); \ (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \
pos = rht_dereference_bucket((pos)->next, tbl, hash)) pos = rht_dereference_bucket((pos)->next, tbl, hash))
/** /**
...@@ -209,9 +246,9 @@ void rhashtable_destroy(struct rhashtable *ht); ...@@ -209,9 +246,9 @@ void rhashtable_destroy(struct rhashtable *ht);
*/ */
#define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member) \ #define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member) \
for (pos = rht_dereference_bucket((tbl)->buckets[hash], tbl, hash), \ for (pos = rht_dereference_bucket((tbl)->buckets[hash], tbl, hash), \
next = pos ? rht_dereference_bucket(pos->next, tbl, hash) \ next = !rht_is_a_nulls(pos) ? \
: NULL; \ rht_dereference_bucket(pos->next, tbl, hash) : NULL; \
pos && rht_entry(tpos, pos, member); \ (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \
pos = next) pos = next)
/** /**
...@@ -228,7 +265,7 @@ void rhashtable_destroy(struct rhashtable *ht); ...@@ -228,7 +265,7 @@ void rhashtable_destroy(struct rhashtable *ht);
#define rht_for_each_rcu_continue(pos, head, tbl, hash) \ #define rht_for_each_rcu_continue(pos, head, tbl, hash) \
for (({barrier(); }), \ for (({barrier(); }), \
pos = rht_dereference_bucket_rcu(head, tbl, hash); \ pos = rht_dereference_bucket_rcu(head, tbl, hash); \
pos; \ !rht_is_a_nulls(pos); \
pos = rcu_dereference_raw(pos->next)) pos = rcu_dereference_raw(pos->next))
/** /**
...@@ -260,7 +297,7 @@ void rhashtable_destroy(struct rhashtable *ht); ...@@ -260,7 +297,7 @@ void rhashtable_destroy(struct rhashtable *ht);
#define rht_for_each_entry_rcu_continue(tpos, pos, head, tbl, hash, member) \ #define rht_for_each_entry_rcu_continue(tpos, pos, head, tbl, hash, member) \
for (({barrier(); }), \ for (({barrier(); }), \
pos = rht_dereference_bucket_rcu(head, tbl, hash); \ pos = rht_dereference_bucket_rcu(head, tbl, hash); \
pos && rht_entry(tpos, pos, member); \ (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \
pos = rht_dereference_bucket_rcu(pos->next, tbl, hash)) pos = rht_dereference_bucket_rcu(pos->next, tbl, hash))
/** /**
......
...@@ -28,6 +28,9 @@ ...@@ -28,6 +28,9 @@
#define HASH_MIN_SIZE 4UL #define HASH_MIN_SIZE 4UL
#define BUCKET_LOCKS_PER_CPU 128UL #define BUCKET_LOCKS_PER_CPU 128UL
/* Base bits plus 1 bit for nulls marker */
#define HASH_RESERVED_SPACE (RHT_BASE_BITS + 1)
enum { enum {
RHT_LOCK_NORMAL, RHT_LOCK_NORMAL,
RHT_LOCK_NESTED, RHT_LOCK_NESTED,
...@@ -86,7 +89,7 @@ static u32 obj_raw_hashfn(const struct rhashtable *ht, const void *ptr) ...@@ -86,7 +89,7 @@ static u32 obj_raw_hashfn(const struct rhashtable *ht, const void *ptr)
hash = ht->p.hashfn(ptr + ht->p.key_offset, ht->p.key_len, hash = ht->p.hashfn(ptr + ht->p.key_offset, ht->p.key_len,
ht->p.hash_rnd); ht->p.hash_rnd);
return hash; return hash >> HASH_RESERVED_SPACE;
} }
static u32 key_hashfn(struct rhashtable *ht, const void *key, u32 len) static u32 key_hashfn(struct rhashtable *ht, const void *key, u32 len)
...@@ -95,6 +98,7 @@ static u32 key_hashfn(struct rhashtable *ht, const void *key, u32 len) ...@@ -95,6 +98,7 @@ static u32 key_hashfn(struct rhashtable *ht, const void *key, u32 len)
u32 hash; u32 hash;
hash = ht->p.hashfn(key, len, ht->p.hash_rnd); hash = ht->p.hashfn(key, len, ht->p.hash_rnd);
hash >>= HASH_RESERVED_SPACE;
return rht_bucket_index(tbl, hash); return rht_bucket_index(tbl, hash);
} }
...@@ -111,7 +115,7 @@ static struct rhash_head __rcu **bucket_tail(struct bucket_table *tbl, u32 n) ...@@ -111,7 +115,7 @@ static struct rhash_head __rcu **bucket_tail(struct bucket_table *tbl, u32 n)
struct rhash_head __rcu **pprev; struct rhash_head __rcu **pprev;
for (pprev = &tbl->buckets[n]; for (pprev = &tbl->buckets[n];
rht_dereference_bucket(*pprev, tbl, n); !rht_is_a_nulls(rht_dereference_bucket(*pprev, tbl, n));
pprev = &rht_dereference_bucket(*pprev, tbl, n)->next) pprev = &rht_dereference_bucket(*pprev, tbl, n)->next)
; ;
...@@ -164,6 +168,7 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, ...@@ -164,6 +168,7 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
{ {
struct bucket_table *tbl; struct bucket_table *tbl;
size_t size; size_t size;
int i;
size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]); size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]);
tbl = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); tbl = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
...@@ -180,6 +185,9 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, ...@@ -180,6 +185,9 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
return NULL; return NULL;
} }
for (i = 0; i < nbuckets; i++)
INIT_RHT_NULLS_HEAD(tbl->buckets[i], ht, i);
return tbl; return tbl;
} }
...@@ -221,7 +229,7 @@ static void hashtable_chain_unzip(const struct rhashtable *ht, ...@@ -221,7 +229,7 @@ static void hashtable_chain_unzip(const struct rhashtable *ht,
/* Old bucket empty, no work needed. */ /* Old bucket empty, no work needed. */
p = rht_dereference_bucket(old_tbl->buckets[old_hash], old_tbl, p = rht_dereference_bucket(old_tbl->buckets[old_hash], old_tbl,
old_hash); old_hash);
if (!p) if (rht_is_a_nulls(p))
return; return;
new_hash = new_hash2 = head_hashfn(ht, new_tbl, p); new_hash = new_hash2 = head_hashfn(ht, new_tbl, p);
...@@ -252,8 +260,8 @@ static void hashtable_chain_unzip(const struct rhashtable *ht, ...@@ -252,8 +260,8 @@ static void hashtable_chain_unzip(const struct rhashtable *ht,
/* Find the subsequent node which does hash to the same /* Find the subsequent node which does hash to the same
* bucket as node P, or NULL if no such node exists. * bucket as node P, or NULL if no such node exists.
*/ */
next = NULL; INIT_RHT_NULLS_HEAD(next, ht, old_hash);
if (he) { if (!rht_is_a_nulls(he)) {
rht_for_each_continue(he, he->next, old_tbl, old_hash) { rht_for_each_continue(he, he->next, old_tbl, old_hash) {
if (head_hashfn(ht, new_tbl, he) == new_hash) { if (head_hashfn(ht, new_tbl, he) == new_hash) {
next = he; next = he;
...@@ -369,11 +377,15 @@ int rhashtable_expand(struct rhashtable *ht) ...@@ -369,11 +377,15 @@ int rhashtable_expand(struct rhashtable *ht)
*/ */
complete = true; complete = true;
for (old_hash = 0; old_hash < old_tbl->size; old_hash++) { for (old_hash = 0; old_hash < old_tbl->size; old_hash++) {
struct rhash_head *head;
old_bucket_lock = bucket_lock(old_tbl, old_hash); old_bucket_lock = bucket_lock(old_tbl, old_hash);
spin_lock_bh(old_bucket_lock); spin_lock_bh(old_bucket_lock);
hashtable_chain_unzip(ht, new_tbl, old_tbl, old_hash); hashtable_chain_unzip(ht, new_tbl, old_tbl, old_hash);
if (old_tbl->buckets[old_hash] != NULL) head = rht_dereference_bucket(old_tbl->buckets[old_hash],
old_tbl, old_hash);
if (!rht_is_a_nulls(head))
complete = false; complete = false;
spin_unlock_bh(old_bucket_lock); spin_unlock_bh(old_bucket_lock);
...@@ -498,6 +510,7 @@ static void rht_deferred_worker(struct work_struct *work) ...@@ -498,6 +510,7 @@ static void rht_deferred_worker(struct work_struct *work)
void rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj) void rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj)
{ {
struct bucket_table *tbl; struct bucket_table *tbl;
struct rhash_head *head;
spinlock_t *lock; spinlock_t *lock;
unsigned hash; unsigned hash;
...@@ -508,7 +521,12 @@ void rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj) ...@@ -508,7 +521,12 @@ void rhashtable_insert(struct rhashtable *ht, struct rhash_head *obj)
lock = bucket_lock(tbl, hash); lock = bucket_lock(tbl, hash);
spin_lock_bh(lock); spin_lock_bh(lock);
RCU_INIT_POINTER(obj->next, tbl->buckets[hash]); head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);
if (rht_is_a_nulls(head))
INIT_RHT_NULLS_HEAD(obj->next, ht, hash);
else
RCU_INIT_POINTER(obj->next, head);
rcu_assign_pointer(tbl->buckets[hash], obj); rcu_assign_pointer(tbl->buckets[hash], obj);
spin_unlock_bh(lock); spin_unlock_bh(lock);
...@@ -709,6 +727,7 @@ static size_t rounded_hashtable_size(struct rhashtable_params *params) ...@@ -709,6 +727,7 @@ static size_t rounded_hashtable_size(struct rhashtable_params *params)
* .key_offset = offsetof(struct test_obj, key), * .key_offset = offsetof(struct test_obj, key),
* .key_len = sizeof(int), * .key_len = sizeof(int),
* .hashfn = jhash, * .hashfn = jhash,
* .nulls_base = (1U << RHT_BASE_SHIFT),
* }; * };
* *
* Configuration Example 2: Variable length keys * Configuration Example 2: Variable length keys
...@@ -741,6 +760,9 @@ int rhashtable_init(struct rhashtable *ht, struct rhashtable_params *params) ...@@ -741,6 +760,9 @@ int rhashtable_init(struct rhashtable *ht, struct rhashtable_params *params)
(!params->key_len && !params->obj_hashfn)) (!params->key_len && !params->obj_hashfn))
return -EINVAL; return -EINVAL;
if (params->nulls_base && params->nulls_base < (1U << RHT_BASE_SHIFT))
return -EINVAL;
params->min_shift = max_t(size_t, params->min_shift, params->min_shift = max_t(size_t, params->min_shift,
ilog2(HASH_MIN_SIZE)); ilog2(HASH_MIN_SIZE));
...@@ -974,6 +996,7 @@ static int __init test_rht_init(void) ...@@ -974,6 +996,7 @@ static int __init test_rht_init(void)
.key_offset = offsetof(struct test_obj, value), .key_offset = offsetof(struct test_obj, value),
.key_len = sizeof(int), .key_len = sizeof(int),
.hashfn = jhash, .hashfn = jhash,
.nulls_base = (3U << RHT_BASE_SHIFT),
.grow_decision = rht_grow_above_75, .grow_decision = rht_grow_above_75,
.shrink_decision = rht_shrink_below_30, .shrink_decision = rht_shrink_below_30,
}; };
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment