[RFC PATCH v2] Broadcast qdisc statistics via netlink on packet dequeue.

Toke Høiland-Jørgensen toke at toke.dk
Mon Jun 17 15:07:27 EDT 2013


This is the second attempt at a patch to broadcast qdisc statistics via
netlink on packet dequeue.

Changes since the first version:

- Cleaned up the patch itself.

- New enum entries in rtnetlink.h are now at the bottom.

- Clean up buffers properly on failure.

- Check if any clients are listening to the netlink group and abort if
  none are.

- Use the non-blocking netlink API (I think...)

- Call the main stats broadcast function from sch_direct_xmit(). Finally
  makes things work the way they're supposed to (i.e. it gets called on
  every packet dequeue).

Client implementation is at https://github.com/tohojo/qstatsc

Preliminary testing on a virtual machine between host and guests gives
similar network performance and similar CPU load with and without the
patch. Still, I wouldn't recommend turning it on at 10GigE speeds.

Comments greatly appreciated.

-Toke

----- >8 ----------- >8 -----
Add qdisc_stats_broadcast_interval sysctl parameter, and use it to limit
stats broadcast interval.
---
 include/net/netns/ipv4.h       |  4 ++
 include/net/sch_generic.h      |  4 ++
 include/uapi/linux/rtnetlink.h |  5 +++
 net/core/gen_stats.c           |  6 ++-
 net/ipv4/sysctl_net_ipv4.c     | 13 ++++++
 net/sched/Kconfig              | 13 ++++++
 net/sched/sch_api.c            |  7 ++++
 net/sched/sch_generic.c        | 89 ++++++++++++++++++++++++++++++++++++++++++
 8 files changed, 139 insertions(+), 2 deletions(-)

diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h
index 2ba9de8..ff69564 100644
--- a/include/net/netns/ipv4.h
+++ b/include/net/netns/ipv4.h
@@ -64,6 +64,10 @@ struct netns_ipv4 {
 
 	int sysctl_tcp_ecn;
 
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+	int sysctl_qdisc_stats_broadcast_interval;
+#endif
+
 	kgid_t sysctl_ping_group_range[2];
 	long sysctl_tcp_mem[3];
 
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index f10818f..92f26cf 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -78,6 +78,10 @@ struct Qdisc {
 	struct netdev_queue	*dev_queue;
 	struct Qdisc		*next_sched;
 
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+	u64 			last_stats_broadcast;
+#endif
+
 	struct sk_buff		*gso_skb;
 	/*
 	 * For performance sake on SMP, we put highly modified fields at the end
diff --git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h
index da0a60e..edee5a9 100644
--- a/include/uapi/linux/rtnetlink.h
+++ b/include/uapi/linux/rtnetlink.h
@@ -132,6 +132,9 @@ enum {
 	RTM_GETMDB = 86,
 #define RTM_GETMDB RTM_GETMDB
 
+	RTM_QDISC_STATS,
+#define RTM_QDISC_STATS RTM_QDISC_STATS
+
 	__RTM_MAX,
 #define RTM_MAX		(((__RTM_MAX + 3) & ~3) - 1)
 };
@@ -613,6 +616,8 @@ enum rtnetlink_groups {
 #define RTNLGRP_IPV6_NETCONF	RTNLGRP_IPV6_NETCONF
 	RTNLGRP_MDB,
 #define RTNLGRP_MDB		RTNLGRP_MDB
+	RTNLGRP_TC_STATS,
+#define RTNLGRP_TC_STATS	RTNLGRP_TC_STATS
 	__RTNLGRP_MAX
 };
 #define RTNLGRP_MAX	(__RTNLGRP_MAX - 1)
diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
index ddedf21..68df614 100644
--- a/net/core/gen_stats.c
+++ b/net/core/gen_stats.c
@@ -61,7 +61,8 @@ gnet_stats_start_copy_compat(struct sk_buff *skb, int type, int tc_stats_type,
 {
 	memset(d, 0, sizeof(*d));
 
-	spin_lock_bh(lock);
+	if(lock)
+		spin_lock_bh(lock);
 	d->lock = lock;
 	if (type)
 		d->tail = (struct nlattr *)skb_tail_pointer(skb);
@@ -245,7 +246,8 @@ gnet_stats_finish_copy(struct gnet_dump *d)
 			return -1;
 	}
 
-	spin_unlock_bh(d->lock);
+	if(d->lock)
+		spin_unlock_bh(d->lock);
 	return 0;
 }
 EXPORT_SYMBOL(gnet_stats_finish_copy);
diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c
index 960fd29..896429c 100644
--- a/net/ipv4/sysctl_net_ipv4.c
+++ b/net/ipv4/sysctl_net_ipv4.c
@@ -851,6 +851,15 @@ static struct ctl_table ipv4_net_table[] = {
 		.mode		= 0644,
 		.proc_handler	= ipv4_tcp_mem,
 	},
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+	{
+		.procname	= "qdisc_stats_broadcast_interval",
+		.data		= &init_net.ipv4.sysctl_qdisc_stats_broadcast_interval,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec
+	},
+#endif
 	{ }
 };
 
@@ -880,6 +889,10 @@ static __net_init int ipv4_sysctl_init_net(struct net *net)
 			&net->ipv4.sysctl_ping_group_range;
 		table[7].data =
 			&net->ipv4.sysctl_tcp_ecn;
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+		table[9].data =
+			&net->ipv4.sysctl_qdisc_stats_broadcast_interval;
+#endif
 
 		/* Don't export sysctls to unprivileged users */
 		if (net->user_ns != &init_user_ns)
diff --git a/net/sched/Kconfig b/net/sched/Kconfig
index 235e01a..03958f0 100644
--- a/net/sched/Kconfig
+++ b/net/sched/Kconfig
@@ -308,6 +308,19 @@ config NET_SCH_PLUG
 	  To compile this code as a module, choose M here: the
 	  module will be called sch_plug.
 
+config NET_SCH_BROADCAST_STATS
+	bool "Enable Qdisc statistics broadcast"
+	---help---
+
+	  Select this option if you want to enable qdisc stats broadcast through
+          netlink multicast. Broadcast happens on packet dequeue, limited to the
+          interval set by the qdisc_stats_broadcast_interval sysctl parameter.
+
+          The statistics will be broadcast to the RTNLGRP_TC_STATS multicast group
+          and the message type is RTM_QDISC_STATS.
+
+          See https://github.com/tohojo/qstatsc for a sample client implementation.
+
 comment "Classification"
 
 config NET_CLS
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index c297e2a..154b316 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -1772,6 +1772,10 @@ static int __net_init psched_net_init(struct net *net)
 	if (e == NULL)
 		return -ENOMEM;
 
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+	net->ipv4.sysctl_qdisc_stats_broadcast_interval = 200000;
+#endif
+
 	return 0;
 }
 
@@ -1782,6 +1786,9 @@ static void __net_exit psched_net_exit(struct net *net)
 #else
 static int __net_init psched_net_init(struct net *net)
 {
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+	net->ipv4.sysctl_qdisc_stats_broadcast_interval = 200000;
+#endif
 	return 0;
 }
 
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index eac7e0e..b220705 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -28,6 +28,7 @@
 #include <net/sch_generic.h>
 #include <net/pkt_sched.h>
 #include <net/dst.h>
+#include <net/netlink.h>
 
 /* Main transmission queue. */
 
@@ -101,6 +102,88 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb,
 	return ret;
 }
 
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+static inline u64 qdisc_stats_time(void)
+{
+	u64 ns = ktime_to_ns(ktime_get());
+	do_div(ns, NSEC_PER_USEC);
+	return ns;
+}
+
+static int qdisc_broadcast_stats(struct Qdisc *q)
+{
+	struct tcmsg *tcm;
+	struct nlmsghdr *nlh;
+	struct gnet_dump d;
+	struct sk_buff *skb;
+	struct net *net;
+	unsigned char *b;
+	u64 time;
+
+	if(!q->dev_queue || !q->dev_queue->dev)
+		return 0;
+
+	net = dev_net(qdisc_dev(q));
+
+	if(!netlink_has_listeners(net->rtnl, RTNLGRP_TC_STATS))
+		return 0;
+
+	time = qdisc_stats_time();
+	if(time < q->last_stats_broadcast +
+		net->ipv4.sysctl_qdisc_stats_broadcast_interval)
+		return 0;
+
+	skb = alloc_skb(NLMSG_SPACE(1024), GFP_ATOMIC);
+	if(!skb)
+		return -ENOBUFS;
+	b = skb_tail_pointer(skb);
+
+	nlh = nlmsg_put(skb, 0, 0, RTM_QDISC_STATS, sizeof(*tcm), NLM_F_MULTI);
+	if (!nlh)
+		goto out_free;
+
+	tcm = nlmsg_data(nlh);
+	tcm->tcm_family = AF_UNSPEC;
+	tcm->tcm__pad1 = 0;
+	tcm->tcm__pad2 = 0;
+	tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
+	tcm->tcm_parent = q->parent;
+	tcm->tcm_handle = q->handle;
+	tcm->tcm_info = atomic_read(&q->refcnt);
+
+	if (nla_put_string(skb, TCA_KIND, q->ops->id))
+		goto nla_put_failure;
+
+	if (gnet_stats_start_copy(skb, TCA_STATS2, NULL, &d) < 0)
+		goto nla_put_failure;
+
+	if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
+		goto nla_put_failure;
+
+	q->qstats.qlen = q->q.qlen;
+	if (gnet_stats_copy_basic(&d, &q->bstats) < 0 ||
+	    gnet_stats_copy_queue(&d, &q->qstats) < 0)
+		goto nla_put_failure;
+
+	if (gnet_stats_finish_copy(&d) < 0)
+		goto nla_put_failure;
+
+	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
+
+	nlmsg_notify(net->rtnl, skb, 0, RTNLGRP_TC_STATS, 0, 0);
+
+	q->last_stats_broadcast = time;
+
+	return 0;
+
+nla_put_failure:
+out_free:
+	kfree_skb(skb);
+	return -1;
+
+}
+#endif
+
 /*
  * Transmit one skb, and handle the return status as required. Holding the
  * __QDISC_STATE_RUNNING bit guarantees that only one CPU can execute this
@@ -115,6 +198,9 @@ int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
 		    spinlock_t *root_lock)
 {
 	int ret = NETDEV_TX_BUSY;
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+	qdisc_broadcast_stats(q);
+#endif
 
 	/* And release qdisc */
 	spin_unlock(root_lock);
@@ -565,6 +651,9 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
 	sch->enqueue = ops->enqueue;
 	sch->dequeue = ops->dequeue;
 	sch->dev_queue = dev_queue;
+#ifdef CONFIG_NET_SCH_BROADCAST_STATS
+	sch->last_stats_broadcast = qdisc_stats_time();
+#endif
 	dev_hold(dev);
 	atomic_set(&sch->refcnt, 1);
 
-- 
1.8.3.1

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 489 bytes
Desc: not available
URL: <https://lists.bufferbloat.net/pipermail/bloat-devel/attachments/20130617/1d8da425/attachment.sig>


More information about the Bloat-devel mailing list