[Cake] dual-src/dsthost unfairness, only with bi-directional traffic
George Amanakis
gamanakis at gmail.com
Tue Jan 15 22:34:30 EST 2019
A better version of the patch for testing.
Setup:
IP{1,2}(flent) <----> Router <----> Server(netserver)
Router:
tc qdisc add dev enp1s0 root cake bandwidth 100mbit dual-srchost besteffort
tc qdisc add dev enp4s0 root cake bandwidth 100mbit dual-dsthost besteffort
IP1:
Data file written to ./tcp_8down-2019-01-15T222742.358874.flent.gz.
Summary of tcp_8down test run at 2019-01-16 03:27:42.358874:
avg median # data pts
Ping (ms) ICMP : 0.86 0.78 ms 342
TCP download avg : 6.16 5.86 Mbits/s 301
TCP download sum : 49.28 46.90 Mbits/s 301
TCP download::1 : 6.23 5.86 Mbits/s 297
TCP download::2 : 6.16 5.87 Mbits/s 297
TCP download::3 : 6.15 5.87 Mbits/s 297
TCP download::4 : 6.14 5.87 Mbits/s 297
TCP download::5 : 6.15 5.87 Mbits/s 297
TCP download::6 : 6.15 5.87 Mbits/s 297
TCP download::7 : 6.15 5.87 Mbits/s 297
TCP download::8 : 6.15 5.87 Mbits/s 297
Data file written to ./tcp_1up-2019-01-15T222743.387906.flent.gz.
Summary of tcp_1up test run at 2019-01-16 03:27:43.387906:
avg median # data pts
Ping (ms) ICMP : 0.87 0.80 ms 343
TCP upload : 47.02 46.20 Mbits/s 265
IP2:
Data file written to ./tcp_1up-2019-01-15T222744.371050.flent.gz.
Summary of tcp_1up test run at 2019-01-16 03:27:44.371050:
avg median # data pts
Ping (ms) ICMP : 0.89 0.77 ms 342
TCP upload : 46.89 46.36 Mbits/s 293
Data file written to ./tcp_8down-2019-01-15T222745.382941.flent.gz.
Summary of tcp_8down test run at 2019-01-16 03:27:45.382941:
avg median # data pts
Ping (ms) ICMP : 0.90 0.81 ms 343
TCP download avg : 6.15 5.86 Mbits/s 301
TCP download sum : 49.23 46.91 Mbits/s 301
TCP download::1 : 6.15 5.87 Mbits/s 297
TCP download::2 : 6.15 5.87 Mbits/s 297
TCP download::3 : 6.15 5.87 Mbits/s 296
TCP download::4 : 6.15 5.87 Mbits/s 297
TCP download::5 : 6.15 5.87 Mbits/s 297
TCP download::6 : 6.16 5.87 Mbits/s 297
TCP download::7 : 6.16 5.87 Mbits/s 297
TCP download::8 : 6.16 5.87 Mbits/s 297
---
sch_cake.c | 67 ++++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 50 insertions(+), 17 deletions(-)
diff --git a/sch_cake.c b/sch_cake.c
index d434ae0..962a090 100644
--- a/sch_cake.c
+++ b/sch_cake.c
@@ -148,6 +148,7 @@ struct cake_host {
u32 dsthost_tag;
u16 srchost_refcnt;
u16 dsthost_refcnt;
+ u16 bulk_flow_count;
};
struct cake_heap_entry {
@@ -1921,12 +1922,22 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
flow->deficit = (b->flow_quantum *
quantum_div[host_load]) >> 16;
} else if (flow->set == CAKE_SET_SPARSE_WAIT) {
+ struct cake_host *srchost = &b->hosts[flow->srchost];
+ struct cake_host *dsthost = &b->hosts[flow->dsthost];
+
/* this flow was empty, accounted as a sparse flow, but actually
* in the bulk rotation.
*/
flow->set = CAKE_SET_BULK;
b->sparse_flow_count--;
b->bulk_flow_count++;
+
+ if (cake_dsrc(q->flow_mode))
+ srchost->bulk_flow_count++;
+
+ if (cake_ddst(q->flow_mode))
+ dsthost->bulk_flow_count++;
+
}
if (q->buffer_used > q->buffer_max_used)
@@ -2097,23 +2108,8 @@ retry:
dsthost = &b->hosts[flow->dsthost];
host_load = 1;
- if (cake_dsrc(q->flow_mode))
- host_load = max(host_load, srchost->srchost_refcnt);
-
- if (cake_ddst(q->flow_mode))
- host_load = max(host_load, dsthost->dsthost_refcnt);
-
- WARN_ON(host_load > CAKE_QUEUES);
-
/* flow isolation (DRR++) */
if (flow->deficit <= 0) {
- /* The shifted prandom_u32() is a way to apply dithering to
- * avoid accumulating roundoff errors
- */
- flow->deficit += (b->flow_quantum * quantum_div[host_load] +
- (prandom_u32() >> 16)) >> 16;
- list_move_tail(&flow->flowchain, &b->old_flows);
-
/* Keep all flows with deficits out of the sparse and decaying
* rotations. No non-empty flow can go into the decaying
* rotation, so they can't get deficits
@@ -2122,6 +2118,13 @@ retry:
if (flow->head) {
b->sparse_flow_count--;
b->bulk_flow_count++;
+
+ if (cake_dsrc(q->flow_mode))
+ srchost->bulk_flow_count++;
+
+ if (cake_ddst(q->flow_mode))
+ dsthost->bulk_flow_count++;
+
flow->set = CAKE_SET_BULK;
} else {
/* we've moved it to the bulk rotation for
@@ -2131,6 +2134,22 @@ retry:
flow->set = CAKE_SET_SPARSE_WAIT;
}
}
+
+ if (cake_dsrc(q->flow_mode))
+ host_load = max(host_load, srchost->bulk_flow_count);
+
+ if (cake_ddst(q->flow_mode))
+ host_load = max(host_load, dsthost->bulk_flow_count);
+
+ WARN_ON(host_load > CAKE_QUEUES);
+
+ /* The shifted prandom_u32() is a way to apply dithering to
+ * avoid accumulating roundoff errors
+ */
+ flow->deficit += (b->flow_quantum * quantum_div[host_load] +
+ (prandom_u32() >> 16)) >> 16;
+ list_move_tail(&flow->flowchain, &b->old_flows);
+
goto retry;
}
@@ -2151,6 +2170,13 @@ retry:
&b->decaying_flows);
if (flow->set == CAKE_SET_BULK) {
b->bulk_flow_count--;
+
+ if (cake_dsrc(q->flow_mode))
+ srchost->bulk_flow_count--;
+
+ if (cake_ddst(q->flow_mode))
+ dsthost->bulk_flow_count--;
+
b->decaying_flow_count++;
} else if (flow->set == CAKE_SET_SPARSE ||
flow->set == CAKE_SET_SPARSE_WAIT) {
@@ -2164,9 +2190,16 @@ retry:
if (flow->set == CAKE_SET_SPARSE ||
flow->set == CAKE_SET_SPARSE_WAIT)
b->sparse_flow_count--;
- else if (flow->set == CAKE_SET_BULK)
+ else if (flow->set == CAKE_SET_BULK) {
b->bulk_flow_count--;
- else
+
+ if (cake_dsrc(q->flow_mode))
+ srchost->bulk_flow_count--;
+
+ if (cake_ddst(q->flow_mode))
+ dsthost->bulk_flow_count--;
+
+ } else
b->decaying_flow_count--;
flow->set = CAKE_SET_NONE;
--
2.20.1
More information about the Cake
mailing list