Re: [Bloat] Progress with latency-under-load tool

[Otto Solares Cabrera <solca@guug.org>]

[-- Attachment #1: Type: text/plain, Size: 2072 bytes --]

On Sun, Mar 20, 2011 at 12:45:10PM +0200, Jonathan Morton wrote:
> Attached is the initial version of the loadlatency tool.  I'm getting some rather interesting results from it already, although it does take a very long time to run.
> 
> It works under Linux, Cygwin and MacOS X on both little- and big-endian machines (and between machines of different byte-sexes), and therefore it should also work under FreeBSD and other UNIXes.  I haven't yet tried compiling it for iOS or Android.
> 
> It produces useful results even when one of the machines is rather old and slow, despite using a proper PRNG for traffic generation.  My ancient Pentium-MMX proved capable of generating at least 4.4 MB/s of traffic steadily, and probably produces spikes of even greater bandwidth.  Anything of Y2K vintage or newer should be able to saturate it's network with this.
> 
> There are four measures produced:  Upload Capacity, Download Capacity, Link Responsiveness and Flow Smoothness.  All of these are reported in "bigger is better" units to help deal with Layers 8 and 9.

Hello Jonathan,

Excellent tool! hopefully with more testing we can validate it's
results and improve it as testing all those scenarios seems the
correct path to understand bufferbloat and latency in our networks.

Sadly it's very difficult for a netadmin like me to follow your "units"
as the world have been standarized on bits per second in base10 or
decimal as opposed to the bytes per second in base2 or binary as
reported by this tool, "smoothness" too is a very radical unit to
measure latency which could be in milliseconds as the tool name implies.

Please don't take it as an offense but I cooked this small patch to
address this problems in the case you want to change it or for others
if they feel the need for more "normal" measuring units which in turn
can lead to more people testing their networks with this tool.

It's a diff against nbd's git repository, it seems to work for me but
honestly I have yet to figure out all the data it reports, hopefully
I didn't b0rk it too much :)
-
 Otto

[-- Attachment #2: loadlatency_units.patch --]
[-- Type: text/x-diff, Size: 6398 bytes --]

--- loadlatency.orig/loadlatency.c	2011-03-23 03:27:16.171982917 -0600
+++ loadlatency/loadlatency.c	2011-03-23 03:43:15.981962124 -0600
@@ -144,7 +144,7 @@
 			}
 
 			if(!settled && numPings > 16 && now-firstPing > 5.0) {
-				printf("MinRTT: %.1fms\n", minRTT * 1000);
+				printf("MinRTT: %.1f ms\n", minRTT * 1000);
 				settled = 1;
 			}
 		} else {
@@ -185,8 +185,8 @@
 typedef struct
 {
 //	uint32_t crc;
-	uint32_t Bps;    // bytes per second, will saturate at 4GB/s per flow.
-	uint32_t smooth; // fixed point 24.8, units are Hz
+	uint32_t bps;     // bits per second, will saturate at 32Gb/s per flow.
+	float    latency; // milliseconds
 } stats_packet;
 
 static void* spew(int sockfd, bool server)
@@ -227,12 +227,12 @@
 		goto bail;
 	}
 //	stats->crc = ntohl(stats->crc);
-	stats->Bps = ntohl(stats->Bps);
-	stats->smooth = ntohl(stats->smooth);
+	stats->bps = ntohl(stats->bps);
+	stats->latency = (float) ntohl((uint32_t) stats->latency);
 //	if(stats->crc != ~crc)
 //		memset(stats, 0, sizeof(stats_packet));
 
-//	printf("Raw stats received: Bps=%u smooth=%u\n", stats->Bps, stats->smooth);
+//	printf("Raw stats received: bps=%u latency=%.1f\n", stats->bps, stats->latency);
 
 bail:
 	close(sockfd);
@@ -244,7 +244,7 @@
 	uint8_t fid = get_flow_id();
 	uint8_t buffer[65536];
 //	uint32_t crc = ~0;
-	uint64_t bytes = 0;
+	uint64_t bits = 0;
 	stats_packet *stats = calloc(sizeof(stats), 1);
 	double startTime = gettime(), finishTime = 0;
 	double now = 0, then = 0;
@@ -262,11 +262,11 @@
 			break;
 
 //		crc = crc32(crc, buffer, rv);
-		bytes += rv;
+		bits += rv*8;
 
-		goodput = bytes / (now - startTime);
+		goodput = bits / (now - startTime);
 
-		if(bytes > 1024*1024) {
+		if(bits*8 > 1000*1000) {
 			double wait = now-then;
 			if(wait > maxWait) {
 				maxWait = wait;
@@ -284,10 +284,10 @@
 		if(goodput > maxGoodput) {
 			maxGoodput = goodput;
 			whenEvent = now;
-//			printf("maxGoodput increased to %.0f KiB/s\n", goodput/1024);
+//			printf("maxGoodput increased to %.0f Kb/s\n", goodput/1000);
 		}
 
-		if(!cancelled && (((now - whenEvent) > 60 && bytes > (maxGoodput * maxRTT * 100) && bytes > (maxGoodput * maxWait * 100)) || (now - startTime) > 600)) {
+		if(!cancelled && (((now - whenEvent) > 60 && bits > (maxGoodput * maxRTT * 100) && bits > (maxGoodput * maxWait * 100)) || (now - startTime) > 600)) {
 			// Maybe time to stop, so signal this as appropriate
 			// carry on receiving data until it stops
 			cancel_flow(fid);
@@ -298,21 +298,21 @@
 
 	if(cancelled && !(chug_mask | spew_mask)) {
 		finishTime = now;
-		goodput = bytes / (finishTime - startTime);
+		goodput = bits / (finishTime - startTime);
 
 //		stats->crc = htonl(~crc);
-		stats->Bps = htonl((uint32_t) goodput);
-		stats->smooth = htonl((uint32_t)(256 / maxWait));
+		stats->bps = htonl((uint32_t) goodput);
+		stats->latency = maxWait;
 
 		if(send(sockfd, stats, sizeof(stats_packet), 0) < sizeof(stats_packet))
-			stats->Bps = stats->smooth = 0;
+			stats->bps = stats->latency = 0;
 
-		stats->Bps = ntohl(stats->Bps);
-		stats->smooth = ntohl(stats->smooth);
+		stats->bps = ntohl(stats->bps);
+//		stats->latency = stats->latency;
 
-//		printf("Raw stats sent: Bps=%u smooth=%u\n", stats->Bps, stats->smooth);
+//		printf("Raw stats sent: bps=%u latency=%.1f\n", stats->bps, stats->latency*1000);
 	} else {
-		stats->Bps = stats->smooth = 0;
+		stats->bps = stats->latency = 0;
 	}
 
 	// drain the connection to make the network quiescent and avoid RST packet bursts
@@ -369,7 +369,7 @@
 	const uint8_t scenario_flows[] = { 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 32, 32, 32, 32, 32, 0 };
 	const uint8_t scenario_uplds[] = { 0, 1, 0, 1, 2, 0, 1, 2, 3, 0, 1, 2, 3, 4,  0,  1, 16, 31, 32, 0 };
 	int scenario, upScenarios = 0, dnScenarios = 0;
-	double flowSmoothness = 0, upCapacity = 0, dnCapacity = 0;
+	double flowLatency = 0, upCapacity = 0, dnCapacity = 0;
 	randctx ctx;
 
 	memset(&hints, 0, sizeof hints);
@@ -421,8 +421,7 @@
 		pthread_t bulk_thread[32] = {0};
 		stats_packet *stats[32] = {NULL};
 		int flow, flows = scenario_flows[scenario], ups = scenario_uplds[scenario], dns = flows-ups;
-		double upCap = 0, dnCap = 0, smoothness = 0;
-		uint32_t worstSmooth = 0;
+		double upCap = 0, dnCap = 0, latency = 0, worstLatency = 0;
 
 		printf("Scenario %u: %u uploads, %u downloads... ", scenario+1, ups, dns);
 		fflush(stdout);
@@ -458,31 +457,31 @@
 			pthread_join(bulk_thread[flow], &stats[flow]);
 
 			if(flow < ups)
-				upCap += 1.0 / stats[flow]->Bps;
+				upCap += 1.0 / stats[flow]->bps;
 			else
-				dnCap += 1.0 / stats[flow]->Bps;
+				dnCap += 1.0 / stats[flow]->bps;
 
-			if(!flow || worstSmooth > stats[flow]->smooth)
-				worstSmooth = stats[flow]->smooth;
+			if(!flow || worstLatency < stats[flow]->latency)
+				worstLatency = stats[flow]->latency;
 		}
 
 		if(ups) {
 			upCap = ups*ups/upCap;
-			printf("%u KiB/s up, ", (uint32_t)(upCap / 1024));
+			printf("%u Kb/s up, ", (uint32_t)(upCap / 1000));
 			upCapacity += 1.0 / upCap;
 			upScenarios++;
 		}
 		if(dns) {
 			dnCap = dns*dns/dnCap;
-			printf("%u KiB/s down, ", (uint32_t)(dnCap / 1024));
+			printf("%u Kb/s down, ", (uint32_t)(dnCap / 1000));
 			dnCapacity += 1.0 / dnCap;
 			dnScenarios++;
 		}
 
-		smoothness = worstSmooth / 256.0;
-		printf("%.2f Hz smoothness\n", smoothness);
-		if(!scenario || smoothness < flowSmoothness)
-			flowSmoothness = smoothness;
+		latency = worstLatency;
+		printf("%.1f ms latency\n", latency*1000);
+		if(!scenario || latency > flowLatency)
+			flowLatency = latency;
 	}
 
 	printf("\nOVERALL:\n");
@@ -490,10 +489,10 @@
 	finished = 1;
 	pthread_join(pingpong_thread, NULL);
 
-	printf("    Upload Capacity: %u KiB/s\n", (unsigned int) floor((upScenarios / 1024.0) / upCapacity));
-	printf("  Download Capacity: %u KiB/s\n", (unsigned int) floor((dnScenarios / 1024.0) / dnCapacity));
-	printf("Link Responsiveness: %u Hz\n", (unsigned int) floor(1.0/maxRTT));
-	printf("    Flow Smoothness: %u Hz\n", (unsigned int) floor(flowSmoothness));
+	printf("    Upload Capacity: %u Kb/s\n", (unsigned int) floor((upScenarios / 1000.0) / upCapacity));
+	printf("  Download Capacity: %u Kb/s\n", (unsigned int) floor((dnScenarios / 1000.0) / dnCapacity));
+	printf("       Link Latency: %.1f ms\n", maxRTT*1000);
+	printf("       Flow Latency: %.1f ms\n", flowLatency*1000);
 }
 
 static void* server_conn(void *arg)