ok, these are the test build I've used so far

2 files changed, 397 insertions, 0 deletions

diff --git a/hiper/Makefile b/hiper/Makefile
new file mode 100644
index 000000000..7989d060a
--- /dev/null
+++ b/hiper/Makefile
@@ -0,0 +1,20 @@
+#
+# Build a little app for the Hiper project
+# During dev at least, we use a static libcurl.
+#
+
+LDFLAGS = -lcrypt -lidn -lssl -lcrypto -ldl -lz -lresolv -L../ares/.libs -lcares
+LIBCURL = -L../lib/.libs/ -lcurl
+CFLAGS = -I../include -g -DHAVE_CURL_MULTI_SOCKET
+
+hiper: hiper.o $(LIBCURL)
+	$(CC) -o $@ $< $(LIBCURL) $(LDFLAGS)
+
+hiper.o: hiper.c
+	$(CC) $(CFLAGS) -c $<
+
+clean:
+	rm hiper.o hiper
+
+$(LIBCURL):
+	(cd ../lib && make)
diff --git a/hiper/hiper.c b/hiper/hiper.c
new file mode 100644
index 000000000..c4f077134
--- /dev/null
+++ b/hiper/hiper.c
@@ -0,0 +1,377 @@
+/*****************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * $Id$
+ *
+ * Connect to N sites simultanouesly and download data.
+ *
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/time.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/poll.h>
+
+#include <curl/curl.h>
+
+/* The number of simultanoues connections/transfers we do */
+#define NCONNECTIONS 2000
+
+/* The least number of connections we are interested in, so when we go below
+   this amount we can just as well stop */
+#define NMARGIN 50
+
+/* Number of loops (seconds) we allow the total download amount and alive
+   connections to remain the same until we bail out. Set this slightly higher
+   when using asynch supported libcurl. */
+#define IDLE_TIME 10
+
+struct globalinfo {
+  size_t dlcounter;
+};
+
+struct connection {
+  CURL *e;
+  int id; /* just a counter for easy browsing */
+  char url[80];
+  size_t dlcounter;
+  struct globalinfo *global;
+};
+
+static size_t
+writecallback(void *ptr, size_t size, size_t nmemb, void *data)
+{
+  size_t realsize = size * nmemb;
+  struct connection *c = (struct connection *)data;
+
+  c->dlcounter += realsize;
+  c->global->dlcounter += realsize;
+
+#if 0
+  printf("%02d: %d, total %d\n",
+         c->id, c->dlcounter, c->global->dlcounter);
+#endif
+  return realsize;
+}
+
+/* return the diff between two timevals, in us */
+static long tvdiff(struct timeval *newer, struct timeval *older)
+{
+  return (newer->tv_sec-older->tv_sec)*1000000+
+    (newer->tv_usec-older->tv_usec);
+}
+
+
+/* store the start time of the program in this variable */
+static struct timeval timer;
+
+static void timer_start(void)
+{
+  /* capture the time of the start moment */
+  gettimeofday(&timer, NULL);
+}
+
+static struct timeval cont; /* at this moment we continued */
+
+int still_running; /* keep number of running handles */
+
+struct conncount {
+  long time_us;
+  long laps;
+  long maxtime;
+};
+
+struct conncount timecount[NCONNECTIONS+1];
+
+static struct timeval timerpause;
+static void timer_pause(void)
+{
+  /* capture the time of the pause moment */
+  gettimeofday(&timerpause, NULL);
+
+  /* If we have a previous continue (all times except the first), we can now
+     store the time for a whole "lap" */
+  if(cont.tv_sec) {
+    long lap;
+
+    lap = tvdiff(&timerpause, &cont);
+
+    timecount[still_running].time_us += lap;
+    timecount[still_running].laps++; /* number of times added */
+
+    if(lap > timecount[still_running].maxtime) {
+      timecount[still_running].maxtime = lap;
+    }
+  }
+}
+
+static long paused; /* amount of us we have been pausing */
+
+static void timer_continue(void)
+{
+  /* Capture the time of the restored operation moment, now calculate how long
+     time we were paused and added that to the 'paused' variable.
+   */
+  gettimeofday(&cont, NULL);
+
+  paused += tvdiff(&cont, &timerpause);
+}
+
+static long total; /* amount of us from start to stop */
+static void timer_stop(void)
+{
+  struct timeval stop;
+  /* Capture the time of the operation stopped moment, now calculate how long
+     time we were running and how much of that pausing.
+   */
+  gettimeofday(&stop, NULL);
+
+  total = tvdiff(&stop, &timer);
+}
+
+struct globalinfo info;
+struct connection conns[NCONNECTIONS];
+
+long selects;
+long selectsalive;
+long timeouts;
+
+long perform;
+long performalive;
+long performselect;
+long topselect;
+
+static void report(void)
+{
+  int i;
+  long active = total - paused;
+  long numdl = 0;
+
+  for(i=0; i < NCONNECTIONS; i++) {
+    if(conns[i].dlcounter)
+      numdl++;
+  }
+
+  printf("Summary from %d simultanoues transfers:\n",
+         NCONNECTIONS);
+
+  printf("Total time %ldus - Paused %ldus = Active %ldus =\n Active/total"
+         " %ldus\n",
+         total, paused, active, active/NCONNECTIONS);
+
+  printf(" Active/(connections that delivered data) = %ldus\n",
+         active/numdl);
+
+  printf("%d out of %d connections provided data\n", numdl, NCONNECTIONS);
+
+  printf("%d calls to curl_multi_perform(), average %d alive. "
+         "Average time: %dus\n",
+         perform, performalive/perform, active/perform);
+
+  printf("%d calls to select(), average %d alive\n",
+         selects, selectsalive/selects);
+  printf(" Average number of readable connections per select() return: %d\n",
+         performselect/selects);
+  printf(" Max number of readable connections for a single select() "
+         "return: %d\n",
+         topselect);
+
+  printf("%ld select() timeouts\n", timeouts);
+
+  for(i=1; i< NCONNECTIONS; i++) {
+    if(timecount[i].laps) {
+      printf("Time %d connections, average %ld max %ld (%ld laps) average/conn: %ld\n",
+             i,
+             timecount[i].time_us/timecount[i].laps,
+             timecount[i].maxtime,
+             timecount[i].laps,
+             (timecount[i].time_us/timecount[i].laps)/i );
+    }
+  }
+}
+
+int main(int argc, char **argv)
+{
+  CURLM *multi_handle;
+  CURLMsg *msg;
+  CURLcode code = CURLE_OK;
+  CURLMcode mcode = CURLM_OK;
+  int rc;
+  int i;
+  FILE *urls;
+  int startindex=0;
+  char buffer[256];
+
+  int prevalive=-1;
+  int prevsamecounter=0;
+  int prevtotal = -1;
+
+  memset(&info, 0, sizeof(struct globalinfo));
+
+  if(argc < 2) {
+    printf("Usage: hiper [file] [start index]\n");
+    return 1;
+  }
+
+  urls = fopen(argv[1], "r");
+  if(!urls)
+    /* failed to open list of urls */
+    return 1;
+
+  if(argc > 2)
+    startindex = atoi(argv[2]);
+
+  if(startindex) {
+    /* Pass this many lines before we start using URLs from the file. On
+       repeated invokes, try using different indexes to avoid torturing the
+       same servers. */
+    while(startindex--) {
+      if(!fgets(buffer, sizeof(buffer), urls))
+        break;
+    }
+  }
+
+  /* init the multi stack */
+  multi_handle = curl_multi_init();
+
+  for(i=0; i< NCONNECTIONS; i++) {
+    CURL *e;
+    char *nl;
+
+    memset(&conns[i], 0, sizeof(struct connection));
+
+    /* read a line from the file of URLs */
+    if(!fgets(conns[i].url, sizeof(conns[i].url), urls))
+      /* failed to read a line */
+      break;
+
+    /* strip off trailing newlines */
+    nl = strchr(conns[i].url, '\n');
+    if(nl)
+      *nl=0; /* cut */
+
+    printf("%d: Add URL %s\n", i, conns[i].url);
+
+    e  = curl_easy_init();
+    conns[i].e = e;
+    conns[i].id = i;
+    conns[i].global = &info;
+
+    curl_easy_setopt(e, CURLOPT_URL, conns[i].url);
+    curl_easy_setopt(e, CURLOPT_WRITEFUNCTION, writecallback);
+    curl_easy_setopt(e, CURLOPT_WRITEDATA, &conns[i]);
+#if 0
+    curl_easy_setopt(e, CURLOPT_VERBOSE, 1);
+    curl_easy_setopt(e, CURLOPT_ERRORBUFFER, errorbuffer);
+#endif
+
+    /* add the easy to the multi */
+    curl_multi_add_handle(multi_handle, e);
+  }
+
+    /* we start some action by calling perform right away */
+  while(CURLM_CALL_MULTI_PERFORM ==
+        curl_multi_perform(multi_handle, &still_running));
+
+  printf("Starting timer!\n");
+  timer_start();
+
+  while(still_running) {
+    struct timeval timeout;
+    int rc; /* select() return code */
+
+    fd_set fdread;
+    fd_set fdwrite;
+    fd_set fdexcep;
+    int maxfd;
+
+    FD_ZERO(&fdread);
+    FD_ZERO(&fdwrite);
+    FD_ZERO(&fdexcep);
+
+    /* set a suitable timeout to play around with */
+    timeout.tv_sec = 0;
+    timeout.tv_usec = 50000;
+
+    /* get file descriptors from the transfers */
+    curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+
+    timer_pause();
+    selects++;
+    selectsalive += still_running;
+    rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
+
+    /* Output this here to make it outside the timer */
+    printf("Running: %d (%d bytes)\n", still_running, info.dlcounter);
+
+    timer_continue();
+
+    switch(rc) {
+    case -1:
+      /* select error */
+      break;
+    case 0:
+      timeouts++;
+    default:
+      /* timeout or readable/writable sockets */
+      do {
+        perform++;
+        performalive += still_running;
+      }
+      while(CURLM_CALL_MULTI_PERFORM ==
+            curl_multi_perform(multi_handle, &still_running));
+
+      performselect += rc;
+      if(rc > topselect)
+        topselect = rc;
+      break;
+    }
+    if(still_running < NMARGIN) {
+      printf("Only %d connections left alive, existing\n",
+             still_running);
+      break;
+    }
+
+    if((prevalive == still_running) && (prevtotal == info.dlcounter) &&
+       info.dlcounter) {
+      /* The same amount of still alive transfers as last lap, increase
+         counter. Only do this if _anything_ has been downloaded since it
+         tends to come here during the initial name lookup phase when using
+         asynch DNS libcurl otherwise.
+       */
+      prevsamecounter++;
+
+      if(prevsamecounter >= IDLE_TIME) {
+        /* for the sake of being efficient, we stop the operation when
+           IDLE_TIME has passed without any bytes transfered */
+        printf("Idle time (%d secs) reached (with %d still claimed alive),"
+               " exiting\n",
+               IDLE_TIME, still_running);
+        break;
+      }
+    }
+    else {
+      prevsamecounter=0;
+    }
+    prevalive = still_running;
+    prevtotal = info.dlcounter;
+  }
+
+  timer_stop();
+
+  curl_multi_cleanup(multi_handle);
+
+  /* cleanup all the easy handles */
+  for(i=0; i< NCONNECTIONS; i++)
+    curl_easy_cleanup(conns[i].e);
+
+  report();
+
+  return code;
+}