Peter Silva introduced CURLOPT_MAX_SEND_SPEED_LARGE and

CURLOPT_MAX_RECV_SPEED_LARGE that limit tha maximum rate libcurl is allowed
to send or receive data. This kind of adds the the command line tool's
option --limit-rate to the library.

The rate limiting logic in the curl app is now removed and is instead
provided by libcurl itself. Transfer rate limiting will now also work for -d
and -F, which it didn't before.

10 files changed, 112 insertions, 124 deletions

diff --git a/CHANGES b/CHANGES
index 8330c5b0a..eee1bb3ac 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,16 @@
 
                                   Changelog
 
+Daniel (20 June 2006)
+- Peter Silva introduced CURLOPT_MAX_SEND_SPEED_LARGE and
+  CURLOPT_MAX_RECV_SPEED_LARGE that limit tha maximum rate libcurl is allowed
+  to send or receive data. This kind of adds the the command line tool's
+  option --limit-rate to the library.
+
+  The rate limiting logic in the curl app is now removed and is instead
+  provided by libcurl itself. Transfer rate limiting will now also work for -d
+  and -F, which it didn't before.
+
 Daniel (19 June 2006)
 - Made -K on a file that couldn't be read cause a warning to be displayed.
 
diff --git a/RELEASE-NOTES b/RELEASE-NOTES
index fa207b2f7..7936724ef 100644
--- a/RELEASE-NOTES
+++ b/RELEASE-NOTES
@@ -11,6 +11,7 @@ Curl and libcurl 7.15.5
 
 This release includes the following changes:
 
+ o added CURLOPT_MAX_SEND_SPEED_LARGE and CURLOPT_MAX_RECV_SPEED_LARGE
  o configure --enable-hidden-symbols
 
 This release includes the following bugfixes:
@@ -19,6 +20,7 @@ This release includes the following bugfixes:
 
 Other curl-related news:
 
+ o cURLpp 0.6.0 was released: http://rrette.com/curlpp.html
  o pycurl-7.15.4 was released: http://pycurl.sf.net
 
 New curl mirrors:
@@ -28,6 +30,6 @@ New curl mirrors:
 This release would not have looked like this without help, code, reports and
 advice from friends like these:
 
- Dan Fandrich
+ Dan Fandrich, Peter Silva
 
         Thanks! (and sorry if I forgot to mention someone)
diff --git a/docs/KNOWN_BUGS b/docs/KNOWN_BUGS
index 928ae49ad..38c657f03 100644
--- a/docs/KNOWN_BUGS
+++ b/docs/KNOWN_BUGS
@@ -103,10 +103,6 @@ may have been fixed since this was written!
   http://curl.haxx.se/bug/view.cgi?id=1004841. How?
   http://curl.haxx.se/mail/lib-2004-08/0182.html
 
-9. --limit-rate using -d or -F does not work. This is because the limit logic
-  is provided by the curl app in its read/write callbacks, and when doing
-  -d/-F the callbacks aren't used! http://curl.haxx.se/bug/view.cgi?id=921395
-
 8. Doing resumed upload over HTTP does not work with '-C -', because curl
   doesn't do a HEAD first to get the initial size. This needs to be done
   manually for HTTP PUT resume to work, and then '-C [index]'.
diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 0183da44c..c4b969577 100644
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -1051,6 +1051,14 @@ for the library to consider it too slow and abort.
 Pass a long as parameter. It contains the time in seconds that the transfer
 should be below the \fICURLOPT_LOW_SPEED_LIMIT\fP for the library to consider
 it too slow and abort.
+.IP CURLOPT_MAX_SEND_SPEED_LARGE
+Pass a curl_off_t as parameter.  If an upload exceeds this speed on cumulative 
+average during the transfer, the transfer will pause to keep the average rate 
+less than or equal to the parameter value.  (default: 0, unlimited)
+.IP CURLOPT_MAX_RECV_SPEED_LARGE
+Pass a curl_off_t as parameter.  If an upload exceeds this speed on cumulative 
+average during the transfer, the transfer will pause to keep the average rate 
+less than or equal to the parameter value.  (default: 0, unlimited)
 .IP CURLOPT_MAXCONNECTS
 Pass a long. The set number will be the persistent connection cache size. The
 set amount will be the maximum amount of simultaneously open connections that
diff --git a/include/curl/curl.h b/include/curl/curl.h
index bd9236ed3..79e71ca0a 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -973,6 +973,11 @@ typedef enum {
      Note that this is used only for SSL certificate processing */
   CINIT(CONV_FROM_UTF8_FUNCTION, FUNCTIONPOINT, 144),
 
+  /* if the connection proceeds too quickly then need to slow it down */
+  /* limit-rate: maximum number of bytes per second to send or receive */
+  CINIT(MAX_SEND_SPEED_LARGE, OFF_T, 145),
+  CINIT(MAX_RECV_SPEED_LARGE, OFF_T, 146),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 
diff --git a/lib/multi.c b/lib/multi.c
index bafadd446..9201402a2 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -68,6 +68,7 @@ typedef enum {
   CURLM_STATE_DOING,       /* sending off the request (part 1) */
   CURLM_STATE_DO_MORE,     /* send off the request (part 2) */
   CURLM_STATE_PERFORM,     /* transfer data */
+  CURLM_STATE_TOOFAST,     /* wait because limit-rate exceeded */
   CURLM_STATE_DONE,        /* post data transfer operation */
   CURLM_STATE_COMPLETED,   /* operation complete */
 
@@ -156,6 +157,7 @@ static void multistate(struct Curl_one_easy *easy, CURLMstate state)
     "DOING",
     "DO_MORE",
     "PERFORM",
+    "TOOFAST",
     "DONE",
     "COMPLETED",
   };
@@ -440,6 +442,7 @@ static int multi_getsock(struct Curl_one_easy *easy,
                          int numsocks)
 {
   switch(easy->state) {
+  case CURLM_STATE_TOOFAST:  /* returns 0, so will not select. */
   default:
     return 0;
 
@@ -771,7 +774,37 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       }
       break;
 
+    case CURLM_STATE_TOOFAST: /* limit-rate exceeded in either direction */
+      /* if both rates are within spec, resume transfer */
+      Curl_pgrsUpdate(easy->easy_conn);
+      if ( ( ( easy->easy_handle->set.max_send_speed == 0 ) ||
+             ( easy->easy_handle->progress.ulspeed <
+               easy->easy_handle->set.max_send_speed ) )  &&
+           ( ( easy->easy_handle->set.max_recv_speed == 0 ) ||
+             ( easy->easy_handle->progress.dlspeed <
+               easy->easy_handle->set.max_recv_speed ) )
+        )
+        multistate(easy, CURLM_STATE_PERFORM);
+
+      break;
+
     case CURLM_STATE_PERFORM:
+
+      /* check if over speed */
+      if ( (  ( easy->easy_handle->set.max_send_speed > 0 ) &&
+              ( easy->easy_handle->progress.ulspeed >
+                easy->easy_handle->set.max_send_speed ) )  ||
+           (  ( easy->easy_handle->set.max_recv_speed > 0 ) &&
+              ( easy->easy_handle->progress.dlspeed >
+                easy->easy_handle->set.max_recv_speed ) )
+        ) {
+        /* Transfer is over the speed limit. Change state.  TODO: Call
+         * Curl_expire() with the time left until we're targeted to be below
+         * the speed limit again. */
+        multistate(easy, CURLM_STATE_TOOFAST );
+        break;
+      }
+
       /* read/write data if it is ready to do so */
       easy->result = Curl_readwrite(easy->easy_conn, &done);
 
@@ -825,6 +858,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         }
       }
       break;
+
     case CURLM_STATE_DONE:
       /* post-transfer command */
       easy->result = Curl_done(&easy->easy_conn, CURLE_OK);
diff --git a/lib/transfer.c b/lib/transfer.c
index 370734a13..0af351042 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -1628,15 +1628,31 @@ Transfer(struct connectdata *conn)
 
     interval_ms = 1 * 1000;
 
-    if(k->keepon & KEEP_READ)
-      fd_read = conn->sockfd;
-    else
-      fd_read = CURL_SOCKET_BAD;
-
-    if(k->keepon & KEEP_WRITE)
-      fd_write = conn->writesockfd;
-    else
+    /* limit-rate logic: if speed exceeds threshold, then do not include fd in
+       select set */
+    if ( (conn->data->set.max_send_speed > 0) &&
+         (conn->data->progress.ulspeed > conn->data->set.max_send_speed) )  {
       fd_write = CURL_SOCKET_BAD;
+      Curl_pgrsUpdate(conn);
+    }
+    else {
+      if(k->keepon & KEEP_WRITE)
+        fd_write = conn->writesockfd;
+      else
+        fd_write = CURL_SOCKET_BAD;
+    }
+
+    if ( (conn->data->set.max_recv_speed > 0) &&
+         (conn->data->progress.dlspeed > conn->data->set.max_recv_speed) ) {
+      fd_read = CURL_SOCKET_BAD;
+      Curl_pgrsUpdate(conn);
+    }
+    else {
+      if(k->keepon & KEEP_READ)
+        fd_read = conn->sockfd;
+      else
+        fd_read = CURL_SOCKET_BAD;
+    }
 
     switch (Curl_select(fd_read, fd_write, interval_ms)) {
     case -1: /* select() error, stop reading */
@@ -1651,6 +1667,7 @@ Transfer(struct connectdata *conn)
       continue;
     case 0:  /* timeout */
     default: /* readable descriptors */
+
       result = Curl_readwrite(conn, &done);
       break;
     }
diff --git a/lib/url.c b/lib/url.c
index 985ade1c5..85537b2ce 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -1039,6 +1039,22 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
      */
     data->set.low_speed_limit=va_arg(param, long);
     break;
+  case CURLOPT_MAX_SEND_SPEED_LARGE:
+    /*
+     * The max speed limit that sends transfer more than
+     * CURLOPT_MAX_SEND_PER_SECOND bytes per second the transfer is
+     * throttled..
+     */
+    data->set.max_send_speed=va_arg(param, curl_off_t);
+    break;
+  case CURLOPT_MAX_RECV_SPEED_LARGE:
+    /*
+     * The max speed limit that sends transfer more than
+     * CURLOPT_MAX_RECV_PER_SECOND bytes per second the transfer is
+     * throttled..
+     */
+    data->set.max_recv_speed=va_arg(param, curl_off_t);
+    break;
   case CURLOPT_LOW_SPEED_TIME:
     /*
      * The low speed time that if transfers are below the set
diff --git a/lib/urldata.h b/lib/urldata.h
index b2cdd8dcb..834741fcc 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1039,6 +1039,8 @@ struct UserDefined {
   curl_off_t infilesize;      /* size of file to upload, -1 means unknown */
   long low_speed_limit; /* bytes/second */
   long low_speed_time;  /* number of seconds */
+  curl_off_t max_send_speed; /* high speed limit in bytes/second for upload */
+  curl_off_t max_recv_speed; /* high speed limit in bytes/second for download */
   curl_off_t set_resume_from;  /* continue [ftp] transfer from here */
   char *cookie;         /* HTTP cookie string to send */
   struct curl_slist *headers; /* linked list of extra headers */
diff --git a/src/main.c b/src/main.c
index 5e9340a2d..3eb11eec8 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2660,9 +2660,7 @@ static size_t my_fwrite(void *buffer, size_t sz, size_t nmemb, void *stream)
   size_t rc;
   struct OutStruct *out=(struct OutStruct *)stream;
   struct Configurable *config = out->config;
-  curl_off_t size = (curl_off_t)(sz * nmemb); /* typecast to prevent
-                                                 warnings when converting from
-                                                 unsigned to signed */
+
   if(out && !out->stream) {
     /* open file for writing */
     out->stream=fopen(out->filename, "wb");
@@ -2679,55 +2677,6 @@ static size_t my_fwrite(void *buffer, size_t sz, size_t nmemb, void *stream)
     }
   }
 
-  if(config->recvpersecond) {
-    /*
-     * We know when we received data the previous time. We know how much data
-     * we get now. Make sure that this is not faster than we are told to run.
-     * If we're faster, sleep a while *before* doing the fwrite() here.
-     */
-
-    struct timeval now;
-    long timediff;
-    long sleep_time;
-
-    static curl_off_t addit = 0;
-
-    now = curlx_tvnow();
-    timediff = curlx_tvdiff(now, config->lastrecvtime); /* milliseconds */
-
-    if((config->recvpersecond > CURL_MAX_WRITE_SIZE) && (timediff < 100) ) {
-      /* If we allow a rather speedy transfer, add this amount for later
-       * checking. Also, do not modify the lastrecvtime as we will use a
-       * longer scope due to this addition.  We wait for at least 100 ms to
-       * pass to get better values to do better math for the sleep. */
-      addit += size;
-    }
-    else {
-      size += addit; /* add up the possibly added bonus rounds from the
-                        zero timediff calls */
-      addit = 0; /* clear the addition pool */
-
-      if( size*1000 > config->recvpersecond*timediff) {
-        /* figure out how many milliseconds to rest */
-        sleep_time = (long)(size*1000/config->recvpersecond - timediff);
-
-        /*
-         * Make sure we don't sleep for so long that we trigger the speed
-         * limit.  This won't limit the bandwidth quite the way we've been
-         * asked to, but at least the transfer has a chance.
-         */
-        if (config->low_speed_time > 0)
-          sleep_time = MIN(sleep_time,(config->low_speed_time * 1000) / 2);
-
-        if(sleep_time > 0) {
-          go_sleep(sleep_time);
-          now = curlx_tvnow();
-        }
-      }
-      config->lastrecvtime = now;
-    }
-  }
-
   rc = fwrite(buffer, sz, nmemb, out->stream);
 
   if((sz * nmemb) == rc) {
@@ -2772,62 +2721,6 @@ static size_t my_fread(void *buffer, size_t sz, size_t nmemb, void *userp)
 {
   size_t rc;
   struct InStruct *in=(struct InStruct *)userp;
-  struct Configurable *config = in->config;
-  curl_off_t size = (curl_off_t)(sz * nmemb);  /* typecast to prevent warnings
-                                                  when converting from
-                                                  unsigned to signed */
-
-  if(config->sendpersecond) {
-    /*
-     * We know when we sent data the previous time. We know how much data
-     * we sent. Make sure that this was not faster than we are told to run.
-     * If we're faster, sleep a while *before* doing the fread() here.
-     * Also, make no larger fread() than should be sent this second!
-     */
-
-    struct timeval now;
-    long timediff;
-    long sleep_time;
-
-    static curl_off_t addit = 0;
-
-    now = curlx_tvnow();
-    timediff = curlx_tvdiff(now, config->lastsendtime); /* milliseconds */
-
-    if((config->sendpersecond > CURL_MAX_WRITE_SIZE) &&
-       (timediff < 100)) {
-      /*
-       * We allow very fast transfers, then allow at least 100 ms between
-       * each sleeping mile-stone to create more accurate long-term rates.
-       */
-      addit += size;
-    }
-    else {
-      /* If 'addit' is non-zero, it contains the total amount of bytes
-         uploaded during the last 'timediff' milliseconds. If it is zero,
-         we use the stored previous size. */
-      curl_off_t xfered = addit?addit:(curl_off_t)config->lastsendsize;
-      addit = 0; /* clear it for the next round */
-
-      if( xfered*1000 > config->sendpersecond*timediff) {
-        /* figure out how many milliseconds to rest */
-        sleep_time = (long)(xfered*1000/config->sendpersecond - timediff);
-        if(sleep_time > 0) {
-          go_sleep (sleep_time);
-          now = curlx_tvnow();
-        }
-      }
-      config->lastsendtime = now;
-
-      if(size > config->sendpersecond) {
-        /* lower the size to actually read */
-        nmemb = (size_t)config->sendpersecond;
-        sz = 1;
-      }
-    }
-
-    config->lastsendsize = sz*nmemb;
-  }
 
   rc = fread(buffer, sz, nmemb, in->stream);
 #if 0
@@ -3890,11 +3783,10 @@ operate(struct Configurable *config, int argc, char *argv[])
         curl_easy_setopt(curl, CURLOPT_IOCTLDATA, &input);
         curl_easy_setopt(curl, CURLOPT_IOCTLFUNCTION, my_ioctl);
 
-        if(config->recvpersecond) {
+        if(config->recvpersecond)
           /* tell libcurl to use a smaller sized buffer as it allows us to
              make better sleeps! 7.9.9 stuff! */
           curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, config->recvpersecond);
-        }
 
         /* size of uploaded file: */
         curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
@@ -3944,8 +3836,14 @@ operate(struct Configurable *config, int argc, char *argv[])
                          config->conf&CONF_AUTO_REFERER);
         curl_easy_setopt(curl, CURLOPT_USERAGENT, config->useragent);
         curl_easy_setopt(curl, CURLOPT_FTPPORT, config->ftpport);
-        curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, config->low_speed_limit);
+        curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT,
+                         config->low_speed_limit);
         curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time);
+        curl_easy_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE,
+                         config->sendpersecond);
+        curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE,
+                         config->recvpersecond);
+
         curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE,
                          config->use_resume?config->resume_from:0);
         curl_easy_setopt(curl, CURLOPT_COOKIE, config->cookie);