Little background, I'm trying to validate some results that I see when using NetPipe to test some connectivity between a couple of linux boxes (over various hardware). So, I concocted this simple client and server to do the same and I cannot seem to get the same numbers as NetPipe - I'm about 30-40% off the rtt times that it sees.
So my question is, is there something stupid that I'm doing wrong with my simple example:
Server:
#include <stdio.h> /* for printf() and fprintf() */
#include <sys/socket.h> /* for socket(), bind(), and connect() */
#include <arpa/inet.h> /* for sockaddr_in and inet_ntoa() */
#include <netinet/tcp.h>
#include <stdlib.h> /* for atoi() and exit() */
#include <string.h> /* for memset() */
#include <unistd.h> /* for close() */
#include <stdio.h> /* for perror() */
#include <stdlib.h> /* for exit() */
#define MAXPENDING 1
void die(char *errorMessage)
{
perror(errorMessage);
exit(1);
}
void handle(unsigned short quickAck, int clntSock)
{
long long c_ts; /* current read timestamp */
int value = 1;
// Enable quickAck
if (quickAck && setsockopt(clntSock, IPPROTO_TCP, TCP_QUICKACK, (char *)&value, sizeof(int)) < 0)
die("TCP_QUICKACK failed");
/* Send received string and receive again until end of transmission */
while (recv(clntSock, (char*)&c_ts, sizeof(c_ts), 0) == sizeof(c_ts)) /* zero indicates end of transmission */
{
// Enable quickAck
if (quickAck && setsockopt(clntSock, IPPROTO_TCP, TCP_QUICKACK, (char *)&value, sizeof(int)) < 0)
die("TCP_QUICKACK failed");
/* Echo message back to client */
if (send(clntSock, (char*)&c_ts, sizeof(c_ts), 0) != sizeof(c_ts))
die("send() failed to send timestamp");
// Enable quickAck
if (quickAck && setsockopt(clntSock, IPPROTO_TCP, TCP_QUICKACK, (char *)&value, sizeof(int)) < 0)
die("TCP_QUICKACK failed");
}
close(clntSock); /* Close client socket */
}
int main(int argc, char *argv[])
{
int servSock; /* Socket descriptor for server */
int clntSock; /* Socket descriptor for client */
struct sockaddr_in echoServAddr; /* Local address */
struct sockaddr_in echoClntAddr; /* Client address */
unsigned short echoServPort; /* Server port */
unsigned short quickAck;
unsigned int clntLen; /* Length of client address data structure */
int value = 1;
if (argc != 3) /* Test for correct number of arguments */
{
fprintf(stderr, "Usage: %s <Server Port> <Quick Ack>\n", argv[0]);
exit(1);
}
echoServPort = atoi(argv[1]); /* First arg: local port */
quickAck = atoi(argv[2]); /* Whether quick ack is enabled or not */
/* Create socket for incoming connections */
if ((servSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)
die("socket() failed");
/* Construct local address structure */
memset(&echoServAddr, 0, sizeof(echoServAddr)); /* Zero out structure */
echoServAddr.sin_family = AF_INET; /* Internet address family */
echoServAddr.sin_addr.s_addr = htonl(INADDR_ANY); /* Any incoming interface */
echoServAddr.sin_port = htons(echoServPort); /* Local port */
/* Bind to the local address */
if (bind(servSock, (struct sockaddr *) &echoServAddr, sizeof(echoServAddr)) < 0)
die("bind() failed");
/* Mark the socket so it will listen for incoming connections */
if (listen(servSock, MAXPENDING) < 0)
die("listen() failed");
for (;;) /* Run forever */
{
/* Set the size of the in-out parameter */
clntLen = sizeof(echoClntAddr);
printf("Waiting for client...\n");
/* Wait for a client to connect */
if ((clntSock = accept(servSock, (struct sockaddr *) &echoClntAddr, &clntLen)) < 0)
die("accept() failed");
/* clntSock is connected to a client! */
printf("Handling client %s\n", inet_ntoa(echoClntAddr.sin_addr));
if (setsockopt(clntSock, IPPROTO_TCP, TCP_NODELAY, (char *)&value, sizeof(int)) < 0)
die("TCP_NODELAY failed");
handle(quickAck, clntSock);
}
/* NOT REACHED */
}
Client:
#include <stdio.h> /* for printf() and fprintf() */
#include <sys/socket.h> /* for socket(), connect(), send(), and recv() */
#include <arpa/inet.h> /* for sockaddr_in and inet_addr() */
#include <netinet/tcp.h>
#include <stdlib.h> /* for atoi() and exit() */
#include <string.h> /* for memset() */
#include <unistd.h> /* for close() */
#include <sys/time.h>
void die(char *errorMessage)
{
perror(errorMessage);
exit(1);
}
int main(int argc, char *argv[])
{
int sock; /* Socket descriptor */
struct sockaddr_in echoServAddr; /* Echo server address */
unsigned short echoServPort; /* Echo server port */
char *servIP; /* Server IP address (dotted quad) */
int iterations, gap, i; /* Number of timestamps to send, and gap between each send */
struct timeval ts;
long long c_ts, o_ts, delta, total = 0, max = 0, min = 1000000000;
int value = 1;
if (argc != 5) /* Test for correct number of arguments */
{
fprintf(stderr, "Usage: %s <Server IP> <Server Port> <Iterations> <Gap>\n", argv[0]);
exit(1);
}
servIP = argv[1]; /* server IP address (dotted quad) */
echoServPort = atoi(argv[2]); /* server port */
iterations = atoi(argv[3]); /* number of timestamps to send */
gap = atoi(argv[4]); /* gap between each send */
/* Create a reliable, stream socket using TCP */
if ((sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)
die("socket() failed");
/* Construct the server address structure */
memset(&echoServAddr, 0, sizeof(echoServAddr)); /* Zero out structure */
echoServAddr.sin_family = AF_INET; /* Internet address family */
echoServAddr.sin_addr.s_addr = inet_addr(servIP); /* Server IP address */
echoServAddr.sin_port = htons(echoServPort); /* Server port */
/* Establish the connection to the echo server */
if (connect(sock, (struct sockaddr *) &echoServAddr, sizeof(echoServAddr)) < 0)
die("connect() failed");
if (setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char *)&value, sizeof(int)) < 0)
die("TCP_NODELAY failed");
/* Give the server a chance */
usleep(1000);
/* Now for the given number of iterations */
for(i = 0; i < iterations; ++i)
{
/* Generate the current timestamp */
gettimeofday(&ts, NULL);
c_ts = ts.tv_sec * 1000000LL + ts.tv_usec;
//printf("sending %ld ", c_ts);
/* Send this */
if (send(sock, (char*)&c_ts, sizeof(c_ts), 0) != sizeof(c_ts))
die("send() failed to send timestamp");
/* Now read the echo */
if (recv(sock, (char*)&o_ts, sizeof(o_ts), 0) != sizeof(o_ts))
die("recv() failed to read timestamp");
gettimeofday(&ts, NULL);
c_ts = ts.tv_sec * 1000000LL + ts.tv_usec;
/* Calculate the delta */
delta = c_ts - o_ts;
//printf(" -> received %ld %ld\n", o_ts, delta);
if (i > 0)
{
/* Track max, min, sum */
total += delta;
max = (max < delta)? delta : max;
min = (min > delta)? delta : min;
}
/* Now sleep */
usleep(1000*gap);
}
--iterations;
printf("iterations %d, avg %f, max %ld, min %ld\n", iterations, (total/(double)iterations), max, min);
close(sock);
exit(0);
}
So, to run, start the server with the port and whether quick_ack is enabled or not (1/0) - this is for a different test. Something like:
./simple_sever 10000 1
Then run the client:
./simple_client <host IP address> 10000 1000 1
So send 1000 timestamps with a 1 millisecond gap between each. This I guess is where the above differs from NetPipe (which floods AFAIK.)
The interaction is pretty straight forward, so is there something I'm missing?
EDIT: Okay got to the bottom of the difference, caching. NetPIPE has an option to force invalidation of cache, and enabling this results in similar numbers to my test program. Phew, I don't have to re-evaluate my sockets programming! I'll leave this question up for reference I guess.
