[Cake] splitting gso at line rate configurably

[Dave Taht <dave.taht@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 2439 bytes --]

so, patches attached. They work. A whole bunch of flent data also
attached. A pretty pic showing how much that over-bql'd burst affected
tcp convergence is also attached....

The BQL reductions at line rate were typically a factor of three which
translates out to the numbers observed here (where a great deal of the
remaining buffering is
in the switch and the stack's context switch itself).

(for the record the lame-arse quad core atom on one side of my testbed
struggles with a local rrul_be test - it ends up with 300000 bytes in
bql with gso enabled)

Toke, would you sign off on this? I'm willing to make the attempt to
upstream it and take the flack from the 50gbit folk, but a sign-off
would be nice (as well as a tactful commit message))

This was as tactful as I could get:

cake: Make gso-splitting configurable

This patch restores cake's behavior at line rate to always split gso,
and makes gso splitting configurable from userspace.

running cake at 1gigE, local traffic:

bql limit: 131966 - no-split-gso
bql limit: ~42392-45420  - split-gso

On a 4 stream test splitting gso apart results in halving the
observed interpacket latency at no effect in throughput.

Summary of tcp_nup test run 'gso-split' (at 2018-07-26 16:03:51.824728):

 Ping (ms) ICMP :         0.83         0.81 ms              341
 TCP upload avg :       235.43       235.39 Mbits/s         301
 TCP upload sum :       941.71       941.56 Mbits/s         301
 TCP upload::1  :       235.45       235.43 Mbits/s         271
 TCP upload::2  :       235.45       235.41 Mbits/s         289
 TCP upload::3  :       235.40       235.40 Mbits/s         288
 TCP upload::4  :       235.41       235.40 Mbits/s         291

vs

Summary of tcp_nup test run 'no-split-gso' (at 2018-07-26 16:37:23.563960):

                           avg       median          # data pts
 Ping (ms) ICMP :         1.67         1.73 ms              348
 TCP upload avg :       234.56       235.37 Mbits/s         301
 TCP upload sum :       938.24       941.49 Mbits/s         301
 TCP upload::1  :       234.55       235.38 Mbits/s         285
 TCP upload::2  :       234.57       235.37 Mbits/s         286
 TCP upload::3  :       234.58       235.37 Mbits/s         274
 TCP upload::4  :       234.54       235.42 Mbits/s         288


-- 

Dave Täht
CEO, TekLibre, LLC
http://www.teklibre.com
Tel: 1-669-226-2619

[-- Attachment #2: iproute2_gso_split.patch --]
[-- Type: text/x-patch, Size: 1645 bytes --]

diff --git a/tc/q_cake.c b/tc/q_cake.c
index f1e232a..727d673 100644
--- a/tc/q_cake.c
+++ b/tc/q_cake.c
@@ -79,6 +79,7 @@ static void explain(void)
 "                  dual-srchost | dual-dsthost | triple-isolate* ]\n"
 "                [ nat | nonat* ]\n"
 "                [ wash | nowash* ]\n"
+"                [ split-gso* | no-split-gso ]\n"
 "                [ ack-filter | ack-filter-aggressive | no-ack-filter* ]\n"
 "                [ memlimit LIMIT ]\n"
 "                [ ptm | atm | noatm* ] [ overhead N | conservative | raw* ]\n"
@@ -108,6 +109,7 @@ static int cake_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 	int nat = -1;
 	int atm = -1;
 	int mpu = 0;
+	int split_gso = -1;
 
 	while (argc > 0) {
 		if (strcmp(*argv, "bandwidth") == 0) {
@@ -155,6 +157,10 @@ static int cake_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 			wash = 0;
 		} else if (strcmp(*argv, "wash") == 0) {
 			wash = 1;
+		} else if (strcmp(*argv, "split-gso") == 0) {
+			split_gso = 1;
+		} else if (strcmp(*argv, "no-split-gso") == 0) {
+			split_gso = 0;
 		} else if (strcmp(*argv, "flowblind") == 0) {
 			flowmode = CAKE_FLOW_NONE;
 		} else if (strcmp(*argv, "srchost") == 0) {
@@ -374,6 +380,9 @@ static int cake_parse_opt(struct qdisc_util *qu, int argc, char **argv,
 		addattr_l(n, 1024, TCA_CAKE_NAT, &nat, sizeof(nat));
 	if (wash != -1)
 		addattr_l(n, 1024, TCA_CAKE_WASH, &wash, sizeof(wash));
+	if (split_gso != -1)
+		addattr_l(n, 1024, TCA_CAKE_SPLIT_GSO, &split_gso,
+			  sizeof(split_gso));
 	if (ingress != -1)
 		addattr_l(n, 1024, TCA_CAKE_INGRESS, &ingress, sizeof(ingress));
 	if (ack_filter != -1)

[-- Attachment #3: kernel_gso_split.patch --]
[-- Type: text/x-patch, Size: 1217 bytes --]

diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c
index 539c949..35fc725 100644
--- a/net/sched/sch_cake.c
+++ b/net/sched/sch_cake.c
@@ -80,7 +80,6 @@
 #define CAKE_QUEUES (1024)
 #define CAKE_FLOW_MASK 63
 #define CAKE_FLOW_NAT_FLAG 64
-#define CAKE_SPLIT_GSO_THRESHOLD (125000000) /* 1Gbps */
 
 /* struct cobalt_params - contains codel and blue parameters
  * @interval:	codel initial drop rate
@@ -2569,10 +2568,12 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt,
 	if (tb[TCA_CAKE_MEMORY])
 		q->buffer_config_limit = nla_get_u32(tb[TCA_CAKE_MEMORY]);
 
-	if (q->rate_bps && q->rate_bps <= CAKE_SPLIT_GSO_THRESHOLD)
-		q->rate_flags |= CAKE_FLAG_SPLIT_GSO;
-	else
-		q->rate_flags &= ~CAKE_FLAG_SPLIT_GSO;
+	if (tb[TCA_CAKE_SPLIT_GSO]) {
+		if (!!nla_get_u32(tb[TCA_CAKE_SPLIT_GSO]))
+			q->rate_flags |= CAKE_FLAG_SPLIT_GSO;
+		else
+			q->rate_flags &= ~CAKE_FLAG_SPLIT_GSO;
+	}
 
 	if (q->tins) {
 		sch_tree_lock(sch);
@@ -2608,7 +2609,7 @@ static int cake_init(struct Qdisc *sch, struct nlattr *opt,
 	q->target   =   5000; /* 5ms: codel RFC argues
 			       * for 5 to 10% of interval
 			       */
-
+	q->rate_flags |= CAKE_FLAG_SPLIT_GSO;
 	q->cur_tin = 0;
 	q->cur_flow  = 0;
 

[-- Attachment #4: rrul_be_-_split-gso-both.png --]
[-- Type: image/png, Size: 346738 bytes --]

[-- Attachment #5: gso_split.tgz --]
[-- Type: application/x-compressed-tar, Size: 1237832 bytes --]