sch_generic.c 15.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * net/sched/sch_generic.c	Generic packet scheduler routines.
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 *
 * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *              Jamal Hadi Salim, <hadi@cyberus.ca> 990601
 *              - Ingress support
 */

#include <linux/bitops.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/init.h>
#include <linux/rcupdate.h>
#include <linux/list.h>
#include <net/pkt_sched.h>

/* Main transmission queue. */

31
/* Modifications to data participating in scheduling must be protected with
32
 * queue->lock spinlock.
33 34 35
 *
 * The idea is the following:
 * - enqueue, dequeue are serialized via top level device
36
 *   spinlock queue->lock.
37
 * - ingress filtering is serialized via top level device
38
 *   spinlock dev->rx_queue.lock.
39
 * - updates to tree and tree walking are only done under the rtnl mutex.
Linus Torvalds's avatar
Linus Torvalds committed
40 41 42
 */

void qdisc_lock_tree(struct net_device *dev)
43
	__acquires(dev->tx_queue.lock)
44
	__acquires(dev->rx_queue.lock)
Linus Torvalds's avatar
Linus Torvalds committed
45
{
46
	spin_lock_bh(&dev->tx_queue.lock);
47
	spin_lock(&dev->rx_queue.lock);
Linus Torvalds's avatar
Linus Torvalds committed
48
}
49
EXPORT_SYMBOL(qdisc_lock_tree);
Linus Torvalds's avatar
Linus Torvalds committed
50 51

void qdisc_unlock_tree(struct net_device *dev)
52
	__releases(dev->rx_queue.lock)
53
	__releases(dev->tx_queue.lock)
Linus Torvalds's avatar
Linus Torvalds committed
54
{
55
	spin_unlock(&dev->rx_queue.lock);
56
	spin_unlock_bh(&dev->tx_queue.lock);
Linus Torvalds's avatar
Linus Torvalds committed
57
}
58
EXPORT_SYMBOL(qdisc_unlock_tree);
Linus Torvalds's avatar
Linus Torvalds committed
59

60 61 62 63 64
static inline int qdisc_qlen(struct Qdisc *q)
{
	return q->q.qlen;
}

65
static inline int dev_requeue_skb(struct sk_buff *skb,
66
				  struct netdev_queue *dev_queue,
67
				  struct Qdisc *q)
68 69
{
	if (unlikely(skb->next))
70
		dev_queue->gso_skb = skb;
71 72
	else
		q->ops->requeue(skb, q);
73

74
	netif_schedule_queue(dev_queue);
75 76 77
	return 0;
}

78 79
static inline struct sk_buff *dequeue_skb(struct netdev_queue *dev_queue,
					  struct Qdisc *q)
80
{
81
	struct sk_buff *skb;
82

83 84
	if ((skb = dev_queue->gso_skb))
		dev_queue->gso_skb = NULL;
85 86 87 88 89 90
	else
		skb = q->dequeue(q);

	return skb;
}

91
static inline int handle_dev_cpu_collision(struct sk_buff *skb,
92
					   struct netdev_queue *dev_queue,
93
					   struct Qdisc *q)
94
{
95
	int ret;
96

97
	if (unlikely(dev_queue->xmit_lock_owner == smp_processor_id())) {
98 99 100 101 102 103
		/*
		 * Same CPU holding the lock. It may be a transient
		 * configuration error, when hard_start_xmit() recurses. We
		 * detect it by checking xmit owner and drop the packet when
		 * deadloop is detected. Return OK to try the next skb.
		 */
104
		kfree_skb(skb);
105 106
		if (net_ratelimit())
			printk(KERN_WARNING "Dead loop on netdevice %s, "
107
			       "fix it urgently!\n", dev_queue->dev->name);
108 109 110 111 112 113 114
		ret = qdisc_qlen(q);
	} else {
		/*
		 * Another cpu is holding lock, requeue & delay xmits for
		 * some time.
		 */
		__get_cpu_var(netdev_rx_stat).cpu_collision++;
115
		ret = dev_requeue_skb(skb, dev_queue, q);
116 117
	}

118
	return ret;
119 120
}

121
/*
122
 * NOTE: Called under queue->lock with locally disabled BH.
123 124
 *
 * __LINK_STATE_QDISC_RUNNING guarantees only one CPU can process this
125
 * device at a time. queue->lock serializes queue accesses for
126
 * this device AND txq->qdisc pointer itself.
127 128 129
 *
 *  netif_tx_lock serializes accesses to device driver.
 *
130
 *  queue->lock and netif_tx_lock are mutually exclusive,
131 132 133 134 135 136 137 138 139
 *  if one is grabbed, another must be free.
 *
 * Note, that this procedure can be called by a watchdog timer
 *
 * Returns to the caller:
 *				0  - queue is empty or throttled.
 *				>0 - queue is not empty.
 *
 */
140
static inline int qdisc_restart(struct netdev_queue *txq)
Linus Torvalds's avatar
Linus Torvalds committed
141
{
142
	struct Qdisc *q = txq->qdisc;
143
	int ret = NETDEV_TX_BUSY;
144 145
	struct net_device *dev;
	struct sk_buff *skb;
Linus Torvalds's avatar
Linus Torvalds committed
146

147
	/* Dequeue packet */
148
	if (unlikely((skb = dequeue_skb(txq, q)) == NULL))
149
		return 0;
150

151 152

	/* And release queue */
153
	spin_unlock(&txq->lock);
154

155 156
	dev = txq->dev;

157
	HARD_TX_LOCK(dev, txq, smp_processor_id());
158 159
	if (!netif_subqueue_stopped(dev, skb))
		ret = dev_hard_start_xmit(skb, dev);
160
	HARD_TX_UNLOCK(dev, txq);
161

162 163
	spin_lock(&txq->lock);
	q = txq->qdisc;
164

165 166 167 168 169 170 171 172
	switch (ret) {
	case NETDEV_TX_OK:
		/* Driver sent out skb successfully */
		ret = qdisc_qlen(q);
		break;

	case NETDEV_TX_LOCKED:
		/* Driver try lock failed */
173
		ret = handle_dev_cpu_collision(skb, txq, q);
174 175 176 177 178 179 180 181
		break;

	default:
		/* Driver returned NETDEV_TX_BUSY - requeue skb */
		if (unlikely (ret != NETDEV_TX_BUSY && net_ratelimit()))
			printk(KERN_WARNING "BUG %s code %d qlen %d\n",
			       dev->name, ret, q->q.qlen);

182
		ret = dev_requeue_skb(skb, txq, q);
183 184
		break;
	}
185

186
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
187 188
}

189
void __qdisc_run(struct netdev_queue *txq)
190
{
191
	struct net_device *dev = txq->dev;
192 193
	unsigned long start_time = jiffies;

194
	while (qdisc_restart(txq)) {
195 196 197 198 199 200 201 202 203
		if (netif_queue_stopped(dev))
			break;

		/*
		 * Postpone processing if
		 * 1. another process needs the CPU;
		 * 2. we've been doing it for too long.
		 */
		if (need_resched() || jiffies != start_time) {
204
			netif_schedule_queue(txq);
205
			break;
206 207
		}
	}
208 209 210 211

	clear_bit(__LINK_STATE_QDISC_RUNNING, &dev->state);
}

Linus Torvalds's avatar
Linus Torvalds committed
212 213 214
static void dev_watchdog(unsigned long arg)
{
	struct net_device *dev = (struct net_device *)arg;
215
	struct netdev_queue *txq = &dev->tx_queue;
Linus Torvalds's avatar
Linus Torvalds committed
216

Herbert Xu's avatar
Herbert Xu committed
217
	netif_tx_lock(dev);
218
	if (txq->qdisc != &noop_qdisc) {
Linus Torvalds's avatar
Linus Torvalds committed
219 220 221 222
		if (netif_device_present(dev) &&
		    netif_running(dev) &&
		    netif_carrier_ok(dev)) {
			if (netif_queue_stopped(dev) &&
223 224 225 226
			    time_after(jiffies, dev->trans_start + dev->watchdog_timeo)) {

				printk(KERN_INFO "NETDEV WATCHDOG: %s: transmit timed out\n",
				       dev->name);
Linus Torvalds's avatar
Linus Torvalds committed
227
				dev->tx_timeout(dev);
228
				WARN_ON_ONCE(1);
Linus Torvalds's avatar
Linus Torvalds committed
229
			}
230
			if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo)))
Linus Torvalds's avatar
Linus Torvalds committed
231 232 233
				dev_hold(dev);
		}
	}
Herbert Xu's avatar
Herbert Xu committed
234
	netif_tx_unlock(dev);
Linus Torvalds's avatar
Linus Torvalds committed
235 236 237 238 239 240 241 242 243

	dev_put(dev);
}

void __netdev_watchdog_up(struct net_device *dev)
{
	if (dev->tx_timeout) {
		if (dev->watchdog_timeo <= 0)
			dev->watchdog_timeo = 5*HZ;
244 245
		if (!mod_timer(&dev->watchdog_timer,
			       round_jiffies(jiffies + dev->watchdog_timeo)))
Linus Torvalds's avatar
Linus Torvalds committed
246 247 248 249 250 251 252 253 254 255 256
			dev_hold(dev);
	}
}

static void dev_watchdog_up(struct net_device *dev)
{
	__netdev_watchdog_up(dev);
}

static void dev_watchdog_down(struct net_device *dev)
{
Herbert Xu's avatar
Herbert Xu committed
257
	netif_tx_lock_bh(dev);
Linus Torvalds's avatar
Linus Torvalds committed
258
	if (del_timer(&dev->watchdog_timer))
259
		dev_put(dev);
Herbert Xu's avatar
Herbert Xu committed
260
	netif_tx_unlock_bh(dev);
Linus Torvalds's avatar
Linus Torvalds committed
261 262
}

263 264 265 266 267 268
/**
 *	netif_carrier_on - set carrier
 *	@dev: network device
 *
 * Device has detected that carrier.
 */
269 270
void netif_carrier_on(struct net_device *dev)
{
Jeff Garzik's avatar
Jeff Garzik committed
271
	if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
272
		linkwatch_fire_event(dev);
Jeff Garzik's avatar
Jeff Garzik committed
273 274 275
		if (netif_running(dev))
			__netdev_watchdog_up(dev);
	}
276
}
277
EXPORT_SYMBOL(netif_carrier_on);
278

279 280 281 282 283 284
/**
 *	netif_carrier_off - clear carrier
 *	@dev: network device
 *
 * Device has detected loss of carrier.
 */
285 286 287 288 289
void netif_carrier_off(struct net_device *dev)
{
	if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state))
		linkwatch_fire_event(dev);
}
290
EXPORT_SYMBOL(netif_carrier_off);
291

Linus Torvalds's avatar
Linus Torvalds committed
292 293 294 295 296
/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
   under all circumstances. It is difficult to invent anything faster or
   cheaper.
 */

297
static int noop_enqueue(struct sk_buff *skb, struct Qdisc * qdisc)
Linus Torvalds's avatar
Linus Torvalds committed
298 299 300 301 302
{
	kfree_skb(skb);
	return NET_XMIT_CN;
}

303
static struct sk_buff *noop_dequeue(struct Qdisc * qdisc)
Linus Torvalds's avatar
Linus Torvalds committed
304 305 306 307
{
	return NULL;
}

308
static int noop_requeue(struct sk_buff *skb, struct Qdisc* qdisc)
Linus Torvalds's avatar
Linus Torvalds committed
309 310
{
	if (net_ratelimit())
311 312
		printk(KERN_DEBUG "%s deferred output. It is buggy.\n",
		       skb->dev->name);
Linus Torvalds's avatar
Linus Torvalds committed
313 314 315 316
	kfree_skb(skb);
	return NET_XMIT_CN;
}

317
struct Qdisc_ops noop_qdisc_ops __read_mostly = {
Linus Torvalds's avatar
Linus Torvalds committed
318 319 320 321 322 323 324 325 326 327 328 329
	.id		=	"noop",
	.priv_size	=	0,
	.enqueue	=	noop_enqueue,
	.dequeue	=	noop_dequeue,
	.requeue	=	noop_requeue,
	.owner		=	THIS_MODULE,
};

struct Qdisc noop_qdisc = {
	.enqueue	=	noop_enqueue,
	.dequeue	=	noop_dequeue,
	.flags		=	TCQ_F_BUILTIN,
330
	.ops		=	&noop_qdisc_ops,
Linus Torvalds's avatar
Linus Torvalds committed
331 332
	.list		=	LIST_HEAD_INIT(noop_qdisc.list),
};
333
EXPORT_SYMBOL(noop_qdisc);
Linus Torvalds's avatar
Linus Torvalds committed
334

335
static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
Linus Torvalds's avatar
Linus Torvalds committed
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
	.id		=	"noqueue",
	.priv_size	=	0,
	.enqueue	=	noop_enqueue,
	.dequeue	=	noop_dequeue,
	.requeue	=	noop_requeue,
	.owner		=	THIS_MODULE,
};

static struct Qdisc noqueue_qdisc = {
	.enqueue	=	NULL,
	.dequeue	=	noop_dequeue,
	.flags		=	TCQ_F_BUILTIN,
	.ops		=	&noqueue_qdisc_ops,
	.list		=	LIST_HEAD_INIT(noqueue_qdisc.list),
};


static const u8 prio2band[TC_PRIO_MAX+1] =
	{ 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1 };

/* 3-band FIFO queue: old style, but should be a bit faster than
   generic prio+fifo combination.
 */

360 361
#define PFIFO_FAST_BANDS 3

362 363
static inline struct sk_buff_head *prio2list(struct sk_buff *skb,
					     struct Qdisc *qdisc)
Linus Torvalds's avatar
Linus Torvalds committed
364 365
{
	struct sk_buff_head *list = qdisc_priv(qdisc);
366 367
	return list + prio2band[skb->priority & TC_PRIO_MAX];
}
Linus Torvalds's avatar
Linus Torvalds committed
368

369
static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc* qdisc)
370 371
{
	struct sk_buff_head *list = prio2list(skb, qdisc);
Linus Torvalds's avatar
Linus Torvalds committed
372

373
	if (skb_queue_len(list) < qdisc_dev(qdisc)->tx_queue_len) {
Linus Torvalds's avatar
Linus Torvalds committed
374
		qdisc->q.qlen++;
375
		return __qdisc_enqueue_tail(skb, qdisc, list);
Linus Torvalds's avatar
Linus Torvalds committed
376
	}
377 378

	return qdisc_drop(skb, qdisc);
Linus Torvalds's avatar
Linus Torvalds committed
379 380
}

381
static struct sk_buff *pfifo_fast_dequeue(struct Qdisc* qdisc)
Linus Torvalds's avatar
Linus Torvalds committed
382 383 384 385
{
	int prio;
	struct sk_buff_head *list = qdisc_priv(qdisc);

386 387
	for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
		if (!skb_queue_empty(list + prio)) {
Linus Torvalds's avatar
Linus Torvalds committed
388
			qdisc->q.qlen--;
389
			return __qdisc_dequeue_head(qdisc, list + prio);
Linus Torvalds's avatar
Linus Torvalds committed
390 391
		}
	}
392

Linus Torvalds's avatar
Linus Torvalds committed
393 394 395
	return NULL;
}

396
static int pfifo_fast_requeue(struct sk_buff *skb, struct Qdisc* qdisc)
Linus Torvalds's avatar
Linus Torvalds committed
397 398
{
	qdisc->q.qlen++;
399
	return __qdisc_requeue(skb, qdisc, prio2list(skb, qdisc));
Linus Torvalds's avatar
Linus Torvalds committed
400 401
}

402
static void pfifo_fast_reset(struct Qdisc* qdisc)
Linus Torvalds's avatar
Linus Torvalds committed
403 404 405 406
{
	int prio;
	struct sk_buff_head *list = qdisc_priv(qdisc);

407
	for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
408 409 410
		__qdisc_reset_queue(qdisc, list + prio);

	qdisc->qstats.backlog = 0;
Linus Torvalds's avatar
Linus Torvalds committed
411 412 413 414 415
	qdisc->q.qlen = 0;
}

static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
{
416
	struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
Linus Torvalds's avatar
Linus Torvalds committed
417 418

	memcpy(&opt.priomap, prio2band, TC_PRIO_MAX+1);
419
	NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
Linus Torvalds's avatar
Linus Torvalds committed
420 421
	return skb->len;

422
nla_put_failure:
Linus Torvalds's avatar
Linus Torvalds committed
423 424 425
	return -1;
}

426
static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
Linus Torvalds's avatar
Linus Torvalds committed
427
{
428
	int prio;
Linus Torvalds's avatar
Linus Torvalds committed
429 430
	struct sk_buff_head *list = qdisc_priv(qdisc);

431 432
	for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
		skb_queue_head_init(list + prio);
Linus Torvalds's avatar
Linus Torvalds committed
433 434 435 436

	return 0;
}

437
static struct Qdisc_ops pfifo_fast_ops __read_mostly = {
Linus Torvalds's avatar
Linus Torvalds committed
438
	.id		=	"pfifo_fast",
439
	.priv_size	=	PFIFO_FAST_BANDS * sizeof(struct sk_buff_head),
Linus Torvalds's avatar
Linus Torvalds committed
440 441 442 443 444 445 446 447 448
	.enqueue	=	pfifo_fast_enqueue,
	.dequeue	=	pfifo_fast_dequeue,
	.requeue	=	pfifo_fast_requeue,
	.init		=	pfifo_fast_init,
	.reset		=	pfifo_fast_reset,
	.dump		=	pfifo_fast_dump,
	.owner		=	THIS_MODULE,
};

449
struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
450
			  struct Qdisc_ops *ops)
Linus Torvalds's avatar
Linus Torvalds committed
451 452 453
{
	void *p;
	struct Qdisc *sch;
454 455
	unsigned int size;
	int err = -ENOBUFS;
Linus Torvalds's avatar
Linus Torvalds committed
456 457

	/* ensure that the Qdisc and the private data are 32-byte aligned */
458 459
	size = QDISC_ALIGN(sizeof(*sch));
	size += ops->priv_size + (QDISC_ALIGNTO - 1);
Linus Torvalds's avatar
Linus Torvalds committed
460

461
	p = kzalloc(size, GFP_KERNEL);
Linus Torvalds's avatar
Linus Torvalds committed
462
	if (!p)
463 464 465
		goto errout;
	sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
	sch->padded = (char *) sch - (char *) p;
Linus Torvalds's avatar
Linus Torvalds committed
466 467 468 469 470 471

	INIT_LIST_HEAD(&sch->list);
	skb_queue_head_init(&sch->q);
	sch->ops = ops;
	sch->enqueue = ops->enqueue;
	sch->dequeue = ops->dequeue;
472
	sch->dev_queue = dev_queue;
473
	dev_hold(qdisc_dev(sch));
Linus Torvalds's avatar
Linus Torvalds committed
474
	atomic_set(&sch->refcnt, 1);
475 476 477

	return sch;
errout:
478
	return ERR_PTR(err);
479 480
}

481 482 483
struct Qdisc * qdisc_create_dflt(struct net_device *dev,
				 struct netdev_queue *dev_queue,
				 struct Qdisc_ops *ops,
484
				 unsigned int parentid)
485 486
{
	struct Qdisc *sch;
487

488
	sch = qdisc_alloc(dev_queue, ops);
489 490
	if (IS_ERR(sch))
		goto errout;
491
	sch->parent = parentid;
492

Linus Torvalds's avatar
Linus Torvalds committed
493 494 495
	if (!ops->init || ops->init(sch, NULL) == 0)
		return sch;

496
	qdisc_destroy(sch);
497
errout:
Linus Torvalds's avatar
Linus Torvalds committed
498 499
	return NULL;
}
500
EXPORT_SYMBOL(qdisc_create_dflt);
Linus Torvalds's avatar
Linus Torvalds committed
501

502
/* Under queue->lock and BH! */
Linus Torvalds's avatar
Linus Torvalds committed
503 504 505

void qdisc_reset(struct Qdisc *qdisc)
{
506
	const struct Qdisc_ops *ops = qdisc->ops;
Linus Torvalds's avatar
Linus Torvalds committed
507 508 509 510

	if (ops->reset)
		ops->reset(qdisc);
}
511
EXPORT_SYMBOL(qdisc_reset);
Linus Torvalds's avatar
Linus Torvalds committed
512

513
/* this is the rcu callback function to clean up a qdisc when there
Linus Torvalds's avatar
Linus Torvalds committed
514 515 516 517 518 519 520 521
 * are no further references to it */

static void __qdisc_destroy(struct rcu_head *head)
{
	struct Qdisc *qdisc = container_of(head, struct Qdisc, q_rcu);
	kfree((char *) qdisc - qdisc->padded);
}

522
/* Under queue->lock and BH! */
Linus Torvalds's avatar
Linus Torvalds committed
523 524 525

void qdisc_destroy(struct Qdisc *qdisc)
{
526
	const struct Qdisc_ops  *ops = qdisc->ops;
Linus Torvalds's avatar
Linus Torvalds committed
527 528

	if (qdisc->flags & TCQ_F_BUILTIN ||
529
	    !atomic_dec_and_test(&qdisc->refcnt))
Linus Torvalds's avatar
Linus Torvalds committed
530 531
		return;

532 533 534 535 536 537
	list_del(&qdisc->list);
	gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est);
	if (ops->reset)
		ops->reset(qdisc);
	if (ops->destroy)
		ops->destroy(qdisc);
Linus Torvalds's avatar
Linus Torvalds committed
538

539
	module_put(ops->owner);
540
	dev_put(qdisc_dev(qdisc));
Linus Torvalds's avatar
Linus Torvalds committed
541 542
	call_rcu(&qdisc->q_rcu, __qdisc_destroy);
}
543
EXPORT_SYMBOL(qdisc_destroy);
Linus Torvalds's avatar
Linus Torvalds committed
544 545 546

void dev_activate(struct net_device *dev)
{
547 548
	struct netdev_queue *txq = &dev->tx_queue;

Linus Torvalds's avatar
Linus Torvalds committed
549 550 551 552 553 554
	/* No queueing discipline is attached to device;
	   create default one i.e. pfifo_fast for devices,
	   which need queueing and noqueue_qdisc for
	   virtual interfaces
	 */

555
	if (txq->qdisc_sleeping == &noop_qdisc) {
Linus Torvalds's avatar
Linus Torvalds committed
556 557
		struct Qdisc *qdisc;
		if (dev->tx_queue_len) {
558
			qdisc = qdisc_create_dflt(dev, txq,
559
						  &pfifo_fast_ops,
560
						  TC_H_ROOT);
Linus Torvalds's avatar
Linus Torvalds committed
561 562 563 564
			if (qdisc == NULL) {
				printk(KERN_INFO "%s: activation failed\n", dev->name);
				return;
			}
565
			list_add_tail(&qdisc->list, &txq->qdisc_list);
Linus Torvalds's avatar
Linus Torvalds committed
566 567 568
		} else {
			qdisc =  &noqueue_qdisc;
		}
569
		txq->qdisc_sleeping = qdisc;
Linus Torvalds's avatar
Linus Torvalds committed
570 571
	}

572 573 574 575
	if (!netif_carrier_ok(dev))
		/* Delay activation until next carrier-on event */
		return;

576 577 578
	spin_lock_bh(&txq->lock);
	rcu_assign_pointer(txq->qdisc, txq->qdisc_sleeping);
	if (txq->qdisc != &noqueue_qdisc) {
Linus Torvalds's avatar
Linus Torvalds committed
579 580 581
		dev->trans_start = jiffies;
		dev_watchdog_up(dev);
	}
582 583 584
	spin_unlock_bh(&txq->lock);
}

585
static void dev_deactivate_queue(struct netdev_queue *dev_queue,
586 587
				 struct Qdisc *qdisc_default)
{
588 589 590 591
	struct Qdisc *qdisc;
	struct sk_buff *skb;

	spin_lock_bh(&dev_queue->lock);
592

593
	qdisc = dev_queue->qdisc;
594 595 596 597
	if (qdisc) {
		dev_queue->qdisc = qdisc_default;
		qdisc_reset(qdisc);
	}
598 599 600 601 602 603
	skb = dev_queue->gso_skb;
	dev_queue->gso_skb = NULL;

	spin_unlock_bh(&dev_queue->lock);

	kfree_skb(skb);
Linus Torvalds's avatar
Linus Torvalds committed
604 605 606 607
}

void dev_deactivate(struct net_device *dev)
{
608
	int running;
Linus Torvalds's avatar
Linus Torvalds committed
609

610
	dev_deactivate_queue(&dev->tx_queue, &noop_qdisc);
611

Linus Torvalds's avatar
Linus Torvalds committed
612 613
	dev_watchdog_down(dev);

614
	/* Wait for outstanding qdisc-less dev_queue_xmit calls. */
615
	synchronize_rcu();
Linus Torvalds's avatar
Linus Torvalds committed
616

617
	/* Wait for outstanding qdisc_run calls. */
618 619 620 621 622 623 624 625
	do {
		while (test_bit(__LINK_STATE_QDISC_RUNNING, &dev->state))
			yield();

		/*
		 * Double-check inside queue lock to ensure that all effects
		 * of the queue run are visible when we return.
		 */
626
		spin_lock_bh(&dev->tx_queue.lock);
627
		running = test_bit(__LINK_STATE_QDISC_RUNNING, &dev->state);
628
		spin_unlock_bh(&dev->tx_queue.lock);
629 630 631 632 633 634 635 636 637 638

		/*
		 * The running flag should never be set at this point because
		 * we've already set dev->qdisc to noop_qdisc *inside* the same
		 * pair of spin locks.  That is, if any qdisc_run starts after
		 * our initial test it should see the noop_qdisc and then
		 * clear the RUNNING bit before dropping the queue lock.  So
		 * if it is set here then we've found a bug.
		 */
	} while (WARN_ON_ONCE(running));
Linus Torvalds's avatar
Linus Torvalds committed
639 640
}

641 642 643 644 645 646 647 648 649
static void dev_init_scheduler_queue(struct net_device *dev,
				     struct netdev_queue *dev_queue,
				     struct Qdisc *qdisc)
{
	dev_queue->qdisc = qdisc;
	dev_queue->qdisc_sleeping = qdisc;
	INIT_LIST_HEAD(&dev_queue->qdisc_list);
}

Linus Torvalds's avatar
Linus Torvalds committed
650 651 652
void dev_init_scheduler(struct net_device *dev)
{
	qdisc_lock_tree(dev);
653 654
	dev_init_scheduler_queue(dev, &dev->tx_queue, &noop_qdisc);
	dev_init_scheduler_queue(dev, &dev->rx_queue, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
655 656
	qdisc_unlock_tree(dev);

657
	setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev);
Linus Torvalds's avatar
Linus Torvalds committed
658 659
}

660 661 662
static void dev_shutdown_scheduler_queue(struct net_device *dev,
					 struct netdev_queue *dev_queue,
					 struct Qdisc *qdisc_default)
Linus Torvalds's avatar
Linus Torvalds committed
663
{
664 665 666 667 668
	struct Qdisc *qdisc = dev_queue->qdisc_sleeping;

	if (qdisc) {
		dev_queue->qdisc = qdisc_default;
		dev_queue->qdisc_sleeping = qdisc_default;
Linus Torvalds's avatar
Linus Torvalds committed
669 670

		qdisc_destroy(qdisc);
671
	}
672 673 674 675 676 677 678
}

void dev_shutdown(struct net_device *dev)
{
	qdisc_lock_tree(dev);
	dev_shutdown_scheduler_queue(dev, &dev->tx_queue, &noop_qdisc);
	dev_shutdown_scheduler_queue(dev, &dev->rx_queue, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
679 680 681
	BUG_TRAP(!timer_pending(&dev->watchdog_timer));
	qdisc_unlock_tree(dev);
}