#include <linux/kmod.h>
#include <linux/list.h>
#include <linux/hrtimer.h>
+#include <linux/lockdep.h>
#include <net/net_namespace.h>
#include <net/sock.h>
(root qdisc, all its children, children of children etc.)
*/
+struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
+{
+ struct Qdisc *q;
+
+ if (!(root->flags & TCQ_F_BUILTIN) &&
+ root->handle == handle)
+ return root;
+
+ list_for_each_entry(q, &root->list, list) {
+ if (q->handle == handle)
+ return q;
+ }
+ return NULL;
+}
+
struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
{
unsigned int i;
for (i = 0; i < dev->num_tx_queues; i++) {
struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
- struct Qdisc *q, *txq_root = txq->qdisc;
-
- if (!(txq_root->flags & TCQ_F_BUILTIN) &&
- txq_root->handle == handle)
- return txq_root;
+ struct Qdisc *q, *txq_root = txq->qdisc_sleeping;
- list_for_each_entry(q, &txq_root->list, list) {
- if (q->handle == handle)
- return q;
- }
+ q = qdisc_match_from_root(txq_root, handle);
+ if (q)
+ return q;
}
- return NULL;
+ return qdisc_match_from_root(dev->rx_queue.qdisc_sleeping, handle);
}
static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
if (!s || tsize != s->tsize || (!tab && tsize > 0))
return ERR_PTR(-EINVAL);
- spin_lock(&qdisc_stab_lock);
+ spin_lock_bh(&qdisc_stab_lock);
list_for_each_entry(stab, &qdisc_stab_list, list) {
if (memcmp(&stab->szopts, s, sizeof(*s)))
if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16)))
continue;
stab->refcnt++;
- spin_unlock(&qdisc_stab_lock);
+ spin_unlock_bh(&qdisc_stab_lock);
return stab;
}
- spin_unlock(&qdisc_stab_lock);
+ spin_unlock_bh(&qdisc_stab_lock);
stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL);
if (!stab)
if (tsize > 0)
memcpy(stab->data, tab, tsize * sizeof(u16));
- spin_lock(&qdisc_stab_lock);
+ spin_lock_bh(&qdisc_stab_lock);
list_add_tail(&stab->list, &qdisc_stab_list);
- spin_unlock(&qdisc_stab_lock);
+ spin_unlock_bh(&qdisc_stab_lock);
return stab;
}
if (!tab)
return;
- spin_lock(&qdisc_stab_lock);
+ spin_lock_bh(&qdisc_stab_lock);
if (--tab->refcnt == 0) {
list_del(&tab->list);
kfree(tab);
}
- spin_unlock(&qdisc_stab_lock);
+ spin_unlock_bh(&qdisc_stab_lock);
}
EXPORT_SYMBOL(qdisc_put_stab);
wd->qdisc->flags &= ~TCQ_F_THROTTLED;
smp_wmb();
- __netif_schedule(wd->qdisc);
+ __netif_schedule(qdisc_root(wd->qdisc));
return HRTIMER_NORESTART;
}
static struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
struct Qdisc *qdisc)
{
+ struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
spinlock_t *root_lock;
- struct Qdisc *oqdisc;
- int ingress;
-
- ingress = 0;
- if (qdisc && qdisc->flags&TCQ_F_INGRESS)
- ingress = 1;
-
- if (ingress) {
- oqdisc = dev_queue->qdisc;
- } else {
- oqdisc = dev_queue->qdisc_sleeping;
- }
root_lock = qdisc_root_lock(oqdisc);
spin_lock_bh(root_lock);
- if (ingress) {
- /* Prune old scheduler */
- if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1) {
- /* delete */
- qdisc_reset(oqdisc);
- dev_queue->qdisc = NULL;
- } else { /* new */
- dev_queue->qdisc = qdisc;
- }
+ /* Prune old scheduler */
+ if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
+ qdisc_reset(oqdisc);
- } else {
- /* Prune old scheduler */
- if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
- qdisc_reset(oqdisc);
-
- /* ... and graft new one */
- if (qdisc == NULL)
- qdisc = &noop_qdisc;
- dev_queue->qdisc_sleeping = qdisc;
- dev_queue->qdisc = &noop_qdisc;
- }
+ /* ... and graft new one */
+ if (qdisc == NULL)
+ qdisc = &noop_qdisc;
+ dev_queue->qdisc_sleeping = qdisc;
+ dev_queue->qdisc = &noop_qdisc;
spin_unlock_bh(root_lock);
if (new || old)
qdisc_notify(skb, n, clid, old, new);
- if (old) {
- spin_lock_bh(&old->q.lock);
+ if (old)
qdisc_destroy(old);
- spin_unlock_bh(&old->q.lock);
- }
}
/* Graft qdisc "new" to class "classid" of qdisc "parent" or
ingress = 0;
num_q = dev->num_tx_queues;
- if (q && q->flags & TCQ_F_INGRESS) {
+ if ((q && q->flags & TCQ_F_INGRESS) ||
+ (new && new->flags & TCQ_F_INGRESS)) {
num_q = 1;
ingress = 1;
}
if (!ingress)
dev_queue = netdev_get_tx_queue(dev, i);
- if (ingress) {
- old = dev_graft_qdisc(dev_queue, q);
- } else {
- old = dev_graft_qdisc(dev_queue, new);
- if (new && i > 0)
- atomic_inc(&new->refcnt);
- }
+ old = dev_graft_qdisc(dev_queue, new);
+ if (new && i > 0)
+ atomic_inc(&new->refcnt);
+
notify_and_destroy(skb, n, classid, old, new);
}
return err;
}
+/* lockdep annotation is needed for ingress; egress gets it only for name */
+static struct lock_class_key qdisc_tx_lock;
+static struct lock_class_key qdisc_rx_lock;
+
/*
Allocate and initialize new qdisc.
if (handle == TC_H_INGRESS) {
sch->flags |= TCQ_F_INGRESS;
handle = TC_H_MAKE(TC_H_INGRESS, 0);
+ lockdep_set_class(qdisc_lock(sch), &qdisc_rx_lock);
} else {
if (handle == 0) {
handle = qdisc_alloc_handle(dev);
if (handle == 0)
goto err_out3;
}
+ lockdep_set_class(qdisc_lock(sch), &qdisc_tx_lock);
}
sch->handle = handle;
goto err_out3;
}
}
- if (parent)
- list_add_tail(&sch->list, &dev_queue->qdisc->list);
+ if ((parent != TC_H_ROOT) && !(sch->flags & TCQ_F_INGRESS))
+ list_add_tail(&sch->list, &dev_queue->qdisc_sleeping->list);
return sch;
}
return -ENOENT;
q = qdisc_leaf(p, clid);
} else { /* ingress */
- q = dev->rx_queue.qdisc;
+ q = dev->rx_queue.qdisc_sleeping;
}
} else {
struct netdev_queue *dev_queue;
return -ENOENT;
q = qdisc_leaf(p, clid);
} else { /*ingress */
- q = dev->rx_queue.qdisc;
+ q = dev->rx_queue.qdisc_sleeping;
}
} else {
struct netdev_queue *dev_queue;
}
graft:
- if (1) {
- spinlock_t *root_lock;
-
- err = qdisc_graft(dev, p, skb, n, clid, q, NULL);
- if (err) {
- if (q) {
- root_lock = qdisc_root_lock(q);
- spin_lock_bh(root_lock);
- qdisc_destroy(q);
- spin_unlock_bh(root_lock);
- }
- return err;
- }
+ err = qdisc_graft(dev, p, skb, n, clid, q, NULL);
+ if (err) {
+ if (q)
+ qdisc_destroy(q);
+ return err;
}
+
return 0;
}
q_idx = 0;
dev_queue = netdev_get_tx_queue(dev, 0);
- if (tc_dump_qdisc_root(dev_queue->qdisc, skb, cb, &q_idx, s_q_idx) < 0)
+ if (tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb, &q_idx, s_q_idx) < 0)
goto done;
dev_queue = &dev->rx_queue;
- if (tc_dump_qdisc_root(dev_queue->qdisc, skb, cb, &q_idx, s_q_idx) < 0)
+ if (tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb, &q_idx, s_q_idx) < 0)
goto done;
cont:
t = 0;
dev_queue = netdev_get_tx_queue(dev, 0);
- if (tc_dump_tclass_root(dev_queue->qdisc, skb, tcm, cb, &t, s_t) < 0)
+ if (tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb, &t, s_t) < 0)
goto done;
dev_queue = &dev->rx_queue;
- if (tc_dump_tclass_root(dev_queue->qdisc, skb, tcm, cb, &t, s_t) < 0)
+ if (tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb, &t, s_t) < 0)
goto done;
done: