Commit efef7939 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'mlx4-next'



Or Gerlitz says:

====================
mlx4 driver update

This series from Matan, Jenny, Dotan and myself is mostly about adding
support to a new performance optimized flow steering mode (patches 4-10).

The 1st two patches are small fixes (one for VXLAN and one for SRIOV),
and the third patch is a fix to avoid hard-lockup situation when many
(hunderds) processes holding user-space QPs/CQs get events.

Matan and Or.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 630f4b70 7d077cd3
......@@ -233,7 +233,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector
if (err)
goto err_dbmap;
cq->mcq.comp = mlx4_ib_cq_comp;
if (context)
cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp;
else
cq->mcq.comp = mlx4_ib_cq_comp;
cq->mcq.event = mlx4_ib_cq_event;
if (context)
......
......@@ -2227,7 +2227,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS;
err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count,
MLX4_IB_UC_STEER_QPN_ALIGN,
&ibdev->steer_qpn_base);
&ibdev->steer_qpn_base, 0);
if (err)
goto err_counter;
......
......@@ -802,16 +802,21 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
}
}
} else {
/* Raw packet QPNs must be aligned to 8 bits. If not, the WQE
* BlueFlame setup flow wrongly causes VLAN insertion. */
/* Raw packet QPNs may not have bits 6,7 set in their qp_num;
* otherwise, the WQE BlueFlame setup flow wrongly causes
* VLAN insertion. */
if (init_attr->qp_type == IB_QPT_RAW_PACKET)
err = mlx4_qp_reserve_range(dev->dev, 1, 1 << 8, &qpn);
err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
(init_attr->cap.max_send_wr ?
MLX4_RESERVE_ETH_BF_QP : 0) |
(init_attr->cap.max_recv_wr ?
MLX4_RESERVE_A0_QP : 0));
else
if (qp->flags & MLX4_IB_QP_NETIF)
err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
else
err = mlx4_qp_reserve_range(dev->dev, 1, 1,
&qpn);
&qpn, 0);
if (err)
goto err_proxy;
}
......
......@@ -76,22 +76,53 @@ void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr)
mlx4_bitmap_free_range(bitmap, obj, 1, use_rr);
}
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align)
static unsigned long find_aligned_range(unsigned long *bitmap,
u32 start, u32 nbits,
int len, int align, u32 skip_mask)
{
unsigned long end, i;
again:
start = ALIGN(start, align);
while ((start < nbits) && (test_bit(start, bitmap) ||
(start & skip_mask)))
start += align;
if (start >= nbits)
return -1;
end = start+len;
if (end > nbits)
return -1;
for (i = start + 1; i < end; i++) {
if (test_bit(i, bitmap) || ((u32)i & skip_mask)) {
start = i + 1;
goto again;
}
}
return start;
}
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt,
int align, u32 skip_mask)
{
u32 obj;
if (likely(cnt == 1 && align == 1))
if (likely(cnt == 1 && align == 1 && !skip_mask))
return mlx4_bitmap_alloc(bitmap);
spin_lock(&bitmap->lock);
obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
bitmap->last, cnt, align - 1);
obj = find_aligned_range(bitmap->table, bitmap->last,
bitmap->max, cnt, align, skip_mask);
if (obj >= bitmap->max) {
bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
& bitmap->mask;
obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
0, cnt, align - 1);
obj = find_aligned_range(bitmap->table, 0, bitmap->max,
cnt, align, skip_mask);
}
if (obj < bitmap->max) {
......@@ -118,6 +149,11 @@ u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap)
return bitmap->avail;
}
static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj)
{
return obj & (bitmap->max + bitmap->reserved_top - 1);
}
void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt,
int use_rr)
{
......@@ -147,6 +183,7 @@ int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,
bitmap->mask = mask;
bitmap->reserved_top = reserved_top;
bitmap->avail = num - reserved_top - reserved_bot;
bitmap->effective_len = bitmap->avail;
spin_lock_init(&bitmap->lock);
bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *
sizeof (long), GFP_KERNEL);
......@@ -163,6 +200,382 @@ void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
kfree(bitmap->table);
}
struct mlx4_zone_allocator {
struct list_head entries;
struct list_head prios;
u32 last_uid;
u32 mask;
/* protect the zone_allocator from concurrent accesses */
spinlock_t lock;
enum mlx4_zone_alloc_flags flags;
};
struct mlx4_zone_entry {
struct list_head list;
struct list_head prio_list;
u32 uid;
struct mlx4_zone_allocator *allocator;
struct mlx4_bitmap *bitmap;
int use_rr;
int priority;
int offset;
enum mlx4_zone_flags flags;
};
struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags)
{
struct mlx4_zone_allocator *zones = kmalloc(sizeof(*zones), GFP_KERNEL);
if (NULL == zones)
return NULL;
INIT_LIST_HEAD(&zones->entries);
INIT_LIST_HEAD(&zones->prios);
spin_lock_init(&zones->lock);
zones->last_uid = 0;
zones->mask = 0;
zones->flags = flags;
return zones;
}
int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc,
struct mlx4_bitmap *bitmap,
u32 flags,
int priority,
int offset,
u32 *puid)
{
u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1);
struct mlx4_zone_entry *it;
struct mlx4_zone_entry *zone = kmalloc(sizeof(*zone), GFP_KERNEL);
if (NULL == zone)
return -ENOMEM;
zone->flags = flags;
zone->bitmap = bitmap;
zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0;
zone->priority = priority;
zone->offset = offset;
spin_lock(&zone_alloc->lock);
zone->uid = zone_alloc->last_uid++;
zone->allocator = zone_alloc;
if (zone_alloc->mask < mask)
zone_alloc->mask = mask;
list_for_each_entry(it, &zone_alloc->prios, prio_list)
if (it->priority >= priority)
break;
if (&it->prio_list == &zone_alloc->prios || it->priority > priority)
list_add_tail(&zone->prio_list, &it->prio_list);
list_add_tail(&zone->list, &it->list);
spin_unlock(&zone_alloc->lock);
*puid = zone->uid;
return 0;
}
/* Should be called under a lock */
static int __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry)
{
struct mlx4_zone_allocator *zone_alloc = entry->allocator;
if (!list_empty(&entry->prio_list)) {
/* Check if we need to add an alternative node to the prio list */
if (!list_is_last(&entry->list, &zone_alloc->entries)) {
struct mlx4_zone_entry *next = list_first_entry(&entry->list,
typeof(*next),
list);
if (next->priority == entry->priority)
list_add_tail(&next->prio_list, &entry->prio_list);
}
list_del(&entry->prio_list);
}
list_del(&entry->list);
if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) {
u32 mask = 0;
struct mlx4_zone_entry *it;
list_for_each_entry(it, &zone_alloc->prios, prio_list) {
u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1);
if (mask < cur_mask)
mask = cur_mask;
}
zone_alloc->mask = mask;
}
return 0;
}
void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc)
{
struct mlx4_zone_entry *zone, *tmp;
spin_lock(&zone_alloc->lock);
list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) {
list_del(&zone->list);
list_del(&zone->prio_list);
kfree(zone);
}
spin_unlock(&zone_alloc->lock);
kfree(zone_alloc);
}
/* Should be called under a lock */
static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count,
int align, u32 skip_mask, u32 *puid)
{
u32 uid;
u32 res;
struct mlx4_zone_allocator *zone_alloc = zone->allocator;
struct mlx4_zone_entry *curr_node;
res = mlx4_bitmap_alloc_range(zone->bitmap, count,
align, skip_mask);
if (res != (u32)-1) {
res += zone->offset;
uid = zone->uid;
goto out;
}
list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) {
if (unlikely(curr_node->priority == zone->priority))
break;
}
if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) {
struct mlx4_zone_entry *it = curr_node;
list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) {
res = mlx4_bitmap_alloc_range(it->bitmap, count,
align, skip_mask);
if (res != (u32)-1) {
res += it->offset;
uid = it->uid;
goto out;
}
}
}
if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) {
struct mlx4_zone_entry *it = curr_node;
list_for_each_entry_from(it, &zone_alloc->entries, list) {
if (unlikely(it == zone))
continue;
if (unlikely(it->priority != curr_node->priority))
break;
res = mlx4_bitmap_alloc_range(it->bitmap, count,
align, skip_mask);
if (res != (u32)-1) {
res += it->offset;
uid = it->uid;
goto out;
}
}
}
if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) {
if (list_is_last(&curr_node->prio_list, &zone_alloc->prios))
goto out;
curr_node = list_first_entry(&curr_node->prio_list,
typeof(*curr_node),
prio_list);
list_for_each_entry_from(curr_node, &zone_alloc->entries, list) {
res = mlx4_bitmap_alloc_range(curr_node->bitmap, count,
align, skip_mask);
if (res != (u32)-1) {
res += curr_node->offset;
uid = curr_node->uid;
goto out;
}
}
}
out:
if (NULL != puid && res != (u32)-1)
*puid = uid;
return res;
}
/* Should be called under a lock */
static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj,
u32 count)
{
mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr);
}
/* Should be called under a lock */
static struct mlx4_zone_entry *__mlx4_find_zone_by_uid(
struct mlx4_zone_allocator *zones, u32 uid)
{
struct mlx4_zone_entry *zone;
list_for_each_entry(zone, &zones->entries, list) {
if (zone->uid == uid)
return zone;
}
return NULL;
}
struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid)
{
struct mlx4_zone_entry *zone;
struct mlx4_bitmap *bitmap;
spin_lock(&zones->lock);
zone = __mlx4_find_zone_by_uid(zones, uid);
bitmap = zone == NULL ? NULL : zone->bitmap;
spin_unlock(&zones->lock);
return bitmap;
}
int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid)
{
struct mlx4_zone_entry *zone;
int res;
spin_lock(&zones->lock);
zone = __mlx4_find_zone_by_uid(zones, uid);
if (NULL == zone) {
res = -1;
goto out;
}
res = __mlx4_zone_remove_one_entry(zone);
out:
spin_unlock(&zones->lock);
kfree(zone);
return res;
}
/* Should be called under a lock */
static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique(
struct mlx4_zone_allocator *zones, u32 obj)
{
struct mlx4_zone_entry *zone, *zone_candidate = NULL;
u32 dist = (u32)-1;
/* Search for the smallest zone that this obj could be
* allocated from. This is done in order to handle
* situations when small bitmaps are allocated from bigger
* bitmaps (and the allocated space is marked as reserved in
* the bigger bitmap.
*/
list_for_each_entry(zone, &zones->entries, list) {
if (obj >= zone->offset) {
u32 mobj = (obj - zone->offset) & zones->mask;
if (mobj < zone->bitmap->max) {
u32 curr_dist = zone->bitmap->effective_len;
if (curr_dist < dist) {
dist = curr_dist;
zone_candidate = zone;
}
}
}
}
return zone_candidate;
}
u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count,
int align, u32 skip_mask, u32 *puid)
{
struct mlx4_zone_entry *zone;
int res = -1;
spin_lock(&zones->lock);
zone = __mlx4_find_zone_by_uid(zones, uid);
if (NULL == zone)
goto out;
res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid);
out:
spin_unlock(&zones->lock);
return res;
}
u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count)
{
struct mlx4_zone_entry *zone;
int res = 0;
spin_lock(&zones->lock);
zone = __mlx4_find_zone_by_uid(zones, uid);
if (NULL == zone) {
res = -1;
goto out;
}
__mlx4_free_from_zone(zone, obj, count);
out:
spin_unlock(&zones->lock);
return res;
}
u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count)
{
struct mlx4_zone_entry *zone;
int res;
if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP))
return -EFAULT;
spin_lock(&zones->lock);
zone = __mlx4_find_zone_by_uid_unique(zones, obj);
if (NULL == zone) {
res = -1;
goto out;
}
__mlx4_free_from_zone(zone, obj, count);
res = 0;
out:
spin_unlock(&zones->lock);
return res;
}
/*
* Handling for queue buffers -- we allocate a bunch of memory and
* register it in a memory region at HCA virtual address 0. If the
......
......@@ -52,6 +52,51 @@
#define MLX4_CQ_STATE_ARMED_SOL ( 6 << 8)
#define MLX4_EQ_STATE_FIRED (10 << 8)
#define TASKLET_MAX_TIME 2
#define TASKLET_MAX_TIME_JIFFIES msecs_to_jiffies(TASKLET_MAX_TIME)
void mlx4_cq_tasklet_cb(unsigned long data)
{
unsigned long flags;
unsigned long end = jiffies + TASKLET_MAX_TIME_JIFFIES;
struct mlx4_eq_tasklet *ctx = (struct mlx4_eq_tasklet *)data;
struct mlx4_cq *mcq, *temp;
spin_lock_irqsave(&ctx->lock, flags);
list_splice_tail_init(&ctx->list, &ctx->process_list);
spin_unlock_irqrestore(&ctx->lock, flags);
list_for_each_entry_safe(mcq, temp, &ctx->process_list, tasklet_ctx.list) {
list_del_init(&mcq->tasklet_ctx.list);
mcq->tasklet_ctx.comp(mcq);
if (atomic_dec_and_test(&mcq->refcount))
complete(&mcq->free);
if (time_after(jiffies, end))
break;
}
if (!list_empty(&ctx->process_list))
tasklet_schedule(&ctx->task);
}
static void mlx4_add_cq_to_tasklet(struct mlx4_cq *cq)
{
unsigned long flags;
struct mlx4_eq_tasklet *tasklet_ctx = cq->tasklet_ctx.priv;
spin_lock_irqsave(&tasklet_ctx->lock, flags);
/* When migrating CQs between EQs will be implemented, please note
* that you need to sync this point. It is possible that
* while migrating a CQ, completions on the old EQs could
* still arrive.
*/
if (list_empty_careful(&cq->tasklet_ctx.list)) {
atomic_inc(&cq->refcount);
list_add_tail(&cq->tasklet_ctx.list, &tasklet_ctx->list);
}
spin_unlock_irqrestore(&tasklet_ctx->lock, flags);
}
void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn)
{
struct mlx4_cq *cq;
......@@ -292,6 +337,11 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
cq->uar = uar;
atomic_set(&cq->refcount, 1);
init_completion(&cq->free);
cq->comp = mlx4_add_cq_to_tasklet;
cq->tasklet_ctx.priv =
&priv->eq_table.eq[cq->vector].tasklet_ctx;
INIT_LIST_HEAD(&cq->tasklet_ctx.list);
cq->irq = priv->eq_table.eq[cq->vector].irq;
return 0;
......
......@@ -595,7 +595,7 @@ static int mlx4_en_get_qp(struct mlx4_en_priv *priv)
return 0;
}
err = mlx4_qp_reserve_range(dev, 1, 1, qpn);
err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP);
en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);
if (err) {
en_err(priv, "Failed to reserve qp for mac registration\n");
......@@ -1974,15 +1974,8 @@ int mlx4_en_alloc_resources(struct mlx4_en_priv *priv)
{
struct mlx4_en_port_profile *prof = priv->prof;
int i;
int err;
int node;
err = mlx4_qp_reserve_range(priv->mdev->dev, priv->tx_ring_num, 256, &priv->base_tx_qpn);
if (err) {