[Cake] dual-src/dsthost unfairness, only with bi-directional traffic

[George Amanakis]

I think what is happening here is that if a client has flows such as "a
(bulk upload)" and "b (bulk download)", the incoming ACKs of flow "a"
compete with the incoming bulk traffic on flow "b". With compete I mean
in terms of flow selection.

So if we adjust the host_load to be the same with the bulk_flow_count of
*each* host, the problem seems to be resolved.
I drafted a patch below.

Pete's setup, tested with the patch (ingress in mbit/s):
IP1: 8down  49.18mbit/s
IP1: 1up    46.73mbit/s
IP2: 1down  47.39mbit/s
IP2: 8up    49.21mbit/s


---
 sch_cake.c | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/sch_cake.c b/sch_cake.c
index d434ae0..5c0f0e1 100644
--- a/sch_cake.c
+++ b/sch_cake.c
@@ -148,6 +148,7 @@ struct cake_host {
 	u32 dsthost_tag;
 	u16 srchost_refcnt;
 	u16 dsthost_refcnt;
+	u16 bulk_flow_count;
 };
 
 struct cake_heap_entry {
@@ -1897,10 +1898,10 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		q->last_packet_time = now;
 	}
 
+	struct cake_host *srchost = &b->hosts[flow->srchost];
+	struct cake_host *dsthost = &b->hosts[flow->dsthost];
 	/* flowchain */
 	if (!flow->set || flow->set == CAKE_SET_DECAYING) {
-		struct cake_host *srchost = &b->hosts[flow->srchost];
-		struct cake_host *dsthost = &b->hosts[flow->dsthost];
 		u16 host_load = 1;
 
 		if (!flow->set) {
@@ -1927,6 +1928,11 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		flow->set = CAKE_SET_BULK;
 		b->sparse_flow_count--;
 		b->bulk_flow_count++;
+		if (cake_dsrc(q->flow_mode))
+			srchost->bulk_flow_count++;
+
+		if (cake_ddst(q->flow_mode))
+			dsthost->bulk_flow_count++;
 	}
 
 	if (q->buffer_used > q->buffer_max_used)
@@ -2101,7 +2107,7 @@ retry:
 		host_load = max(host_load, srchost->srchost_refcnt);
 
 	if (cake_ddst(q->flow_mode))
-		host_load = max(host_load, dsthost->dsthost_refcnt);
+		host_load = max(host_load, dsthost->bulk_flow_count);
 
 	WARN_ON(host_load > CAKE_QUEUES);
 
@@ -2110,8 +2116,6 @@ retry:
 		/* The shifted prandom_u32() is a way to apply dithering to
 		 * avoid accumulating roundoff errors
 		 */
-		flow->deficit += (b->flow_quantum * quantum_div[host_load] +
-				  (prandom_u32() >> 16)) >> 16;
 		list_move_tail(&flow->flowchain, &b->old_flows);
 
 		/* Keep all flows with deficits out of the sparse and decaying
@@ -2122,6 +2126,11 @@ retry:
 			if (flow->head) {
 				b->sparse_flow_count--;
 				b->bulk_flow_count++;
+				if (cake_dsrc(q->flow_mode))
+					srchost->bulk_flow_count++;
+
+				if (cake_ddst(q->flow_mode))
+					dsthost->bulk_flow_count++;
 				flow->set = CAKE_SET_BULK;
 			} else {
 				/* we've moved it to the bulk rotation for
@@ -2131,6 +2140,8 @@ retry:
 				flow->set = CAKE_SET_SPARSE_WAIT;
 			}
 		}
+		flow->deficit += (b->flow_quantum * quantum_div[host_load] +
+				  (prandom_u32() >> 16)) >> 16;
 		goto retry;
 	}
 
@@ -2151,6 +2162,11 @@ retry:
 					       &b->decaying_flows);
 				if (flow->set == CAKE_SET_BULK) {
 					b->bulk_flow_count--;
+					if (cake_dsrc(q->flow_mode))
+						srchost->bulk_flow_count--;
+
+					if (cake_ddst(q->flow_mode))
+						dsthost->bulk_flow_count--;
 					b->decaying_flow_count++;
 				} else if (flow->set == CAKE_SET_SPARSE ||
 					   flow->set == CAKE_SET_SPARSE_WAIT) {
@@ -2164,8 +2180,14 @@ retry:
 				if (flow->set == CAKE_SET_SPARSE ||
 				    flow->set == CAKE_SET_SPARSE_WAIT)
 					b->sparse_flow_count--;
-				else if (flow->set == CAKE_SET_BULK)
+				else if (flow->set == CAKE_SET_BULK) {
 					b->bulk_flow_count--;
+					if (cake_dsrc(q->flow_mode))
+						srchost->bulk_flow_count--;
+
+					if (cake_ddst(q->flow_mode))
+						dsthost->bulk_flow_count--;
+				}
 				else
 					b->decaying_flow_count--;
 
-- 
2.20.1