Vincent Sanders's massive update of this example code. One could argue

weather this is still an "example" or a whole new API layer! ;-)

1 files changed, 496 insertions, 166 deletions

diff --git a/docs/examples/fopen.c b/docs/examples/fopen.c
index 8fc4b1f7c..452995bc3 100644
--- a/docs/examples/fopen.c
+++ b/docs/examples/fopen.c
@@ -1,28 +1,53 @@
 /*****************************************************************************
- *                                  _   _ ____  _     
- *  Project                     ___| | | |  _ \| |    
- *                             / __| | | | |_) | |    
- *                            | (__| |_| |  _ <| |___ 
- *                             \___|\___/|_| \_\_____|
  *
- * $Id$
+ * This example source code introduces a c library buffered I/O interface to
+ * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
+ * rewind(). Supported functions have identical prototypes to their normal c
+ * lib namesakes and are preceaded by url_ .
  *
- * This example source code introduces an fopen()/fread()/fclose() emulation
- * for URL reads. Using an approach similar to this, you could replace your
- * program's fopen() with this url_fopen() and fread() with url_fread() and
- * it should be possible to read remote streams instead of (only) local files.
+ * Using this code you can replace your program's fopen() with url_fopen()
+ * and fread() with url_fread() and it become possible to read remote streams
+ * instead of (only) local files. Local files (ie those that can be directly
+ * fopened) will drop back to using the underlying clib implementations
  *
- * See the main() function at the bottom that shows a tiny app in action.
+ * See the main() function at the bottom that shows an app that retrives from a
+ * specified url using fgets() and fread() and saves as two output files.
  *
- * This source code is a proof of concept. It will need further attention to
- * become production-use useful and solid.
+ * Coyright (c)2003 Simtec Electronics
+ *
+ * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
+ * reference to original curl example code
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * This example requires libcurl 7.9.7 or later.
  */
+
 #include <stdio.h>
 #include <string.h>
 #include <sys/time.h>
 #include <stdlib.h>
+#include <errno.h>
 
 #include <curl/curl.h>
 
@@ -30,206 +55,511 @@
 #error "too old libcurl version, get the latest!"
 #endif
 
-struct data {
-  int type;
-  union {
-    CURL *curl;
-    FILE *file;
-  } handle;
-
-  /* This is the documented biggest possible buffer chunk we can get from
-     libcurl in one single callback! */
-  char buffer[CURL_MAX_WRITE_SIZE];
 
-  char *readptr; /* read from here */
-  int bytes;     /* bytes available from read pointer */
+enum fcurl_type_e { CFTYPE_NONE=0, CFTYPE_FILE=1, CFTYPE_CURL=2 };
 
-  CURLMcode m;   /* stored from a previous url_fread() */
+struct fcurl_data
+{
+    enum fcurl_type_e type;	/* type of handle */
+    union {
+	CURL *curl;
+	FILE *file;
+    } handle;			/* handle */
+
+    char *buffer;		/* buffer to store cached data*/
+    int buffer_len;		/* currently allocated buffers length */
+    int buffer_pos;		/* end of data in buffer*/
+    int still_running;		/* Is background url fetch still in progress */
 };
 
-typedef struct data URL_FILE;
+typedef struct fcurl_data URL_FILE;
+
+/* exported functions */
+URL_FILE *url_fopen(char *url,const char *operation);
+int url_fclose(URL_FILE *file);
+int url_feof(URL_FILE *file);
+size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
+char * url_fgets(char *ptr, int size, URL_FILE *file);
+void url_rewind(URL_FILE *file);
 
 /* we use a global one for convenience */
 CURLM *multi_handle;
 
-static
-size_t write_callback(char *buffer,
-                      size_t size,
-                      size_t nitems,
-                      void *userp)
+/* curl calls this routine to get more data */
+static size_t
+write_callback(char *buffer,
+	       size_t size,
+	       size_t nitems,
+	       void *userp)
 {
-  URL_FILE *url = (URL_FILE *)userp;
-  size *= nitems;
+    char *newbuff;
+    int rembuff;
+
+    URL_FILE *url = (URL_FILE *)userp;
+    size *= nitems;
+
+    rembuff=url->buffer_len - url->buffer_pos;//remaining space in buffer
+
+    if(size > rembuff)
+    {
+	//not enuf space in buffer
+	newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
+	if(newbuff==NULL)
+	{
+	    fprintf(stderr,"callback buffer grow failed\n");
+	    size=rembuff;
+	}
+	else
+	{
+	    /* realloc suceeded increase buffer size*/
+	    url->buffer_len+=size - rembuff;
+	    url->buffer=newbuff;
+
+	    /*printf("Callback buffer grown to %d bytes\n",url->buffer_len);*/
+	}
+    }
+
+    memcpy(&url->buffer[url->buffer_pos], buffer, size);
+    url->buffer_pos += size;
 
-  memcpy(url->readptr, buffer, size);
-  url->readptr += size;
-  url->bytes += size;
+    /*fprintf(stderr, "callback %d size bytes\n", size);*/
+
+    return size;
+}
 
-  fprintf(stderr, "callback %d size bytes\n", size);
+/* use to attempt to fill the read buffer up to requested number of bytes */
+static int
+curl_fill_buffer(URL_FILE *file,int want,int waittime)
+{
+    fd_set fdread;
+    fd_set fdwrite;
+    fd_set fdexcep;
+    int maxfd;
+    struct timeval timeout;
+    int rc;
+
+    /* only attempt to fill buffer if transactions still running and buffer
+     * doesnt exceed required size already
+     */
+    if((!file->still_running) || (file->buffer_pos > want))
+	return 0;
+
+    /* attempt to fill buffer */
+    do
+    {
+	FD_ZERO(&fdread);
+	FD_ZERO(&fdwrite);
+	FD_ZERO(&fdexcep);
+
+	/* set a suitable timeout to fail on */
+	timeout.tv_sec = 60; /* 1 minute */
+	timeout.tv_usec = 0;
+
+	/* get file descriptors from the transfers */
+	curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+
+	rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
+
+	switch(rc) {
+	case -1:
+	    /* select error */
+	    break;
+
+	case 0:
+	    break;
+
+	default:
+	    /* timeout or readable/writable sockets */
+	    /* note we *could* be more efficient and not wait for
+	     * CURLM_CALL_MULTI_PERFORM to clear here and check it on re-entry
+	     * but that gets messy */
+	    while(curl_multi_perform(multi_handle, &file->still_running) ==
+		  CURLM_CALL_MULTI_PERFORM);
+
+	    break;
+	}
+    } while(file->still_running && (file->buffer_pos < want));
+    return 1;
+}
 
-  return size;
+/* use to remove want bytes from the front of a files buffer */
+static int
+curl_use_buffer(URL_FILE *file,int want)
+{
+    /* sort out buffer */
+    if((file->buffer_pos - want) <=0)
+    {
+	/* ditch buffer - write will recreate */
+	if(file->buffer)
+	    free(file->buffer);
+
+	file->buffer=NULL;
+	file->buffer_pos=0;
+	file->buffer_len=0;
+    }
+    else
+    {
+	/* move rest down make it available for later */
+	memmove(file->buffer,
+		&file->buffer[want],
+		(file->buffer_pos - want));
+
+	file->buffer_pos -= want;
+    }
+    return 0;
 }
 
-URL_FILE *url_fopen(char *url, char *operation)
+
+
+URL_FILE *
+url_fopen(char *url,const char *operation)
 {
-  /* this code could check for URLs or types in the 'url' and
-     basicly use the real fopen() for standard files */
+    /* this code could check for URLs or types in the 'url' and
+       basicly use the real fopen() for standard files */
+
+    URL_FILE *file;
+    (void)operation;
+
+    file = (URL_FILE *)malloc(sizeof(URL_FILE));
+    if(!file)
+	return NULL;
 
-  URL_FILE *file;
-  int still_running;
-  (void)operation;
+    memset(file, 0, sizeof(URL_FILE));
+
+    if((file->handle.file=fopen(url,operation)))
+    {
+	file->type = CFTYPE_FILE; /* marked as URL */
+    }
+    else
+    {
+	file->type = CFTYPE_CURL; /* marked as URL */
+	file->handle.curl = curl_easy_init();
 
-  file = (URL_FILE *)malloc(sizeof(URL_FILE));
-  if(!file)
-    return NULL;
+	curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
+	curl_easy_setopt(file->handle.curl, CURLOPT_FILE, file);
+	curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, FALSE);
+	curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
 
-  memset(file, 0, sizeof(URL_FILE));
+	if(!multi_handle)
+	    multi_handle = curl_multi_init();
 
-  file->type = 1; /* marked as URL, use 0 for plain file */
-  file->handle.curl = curl_easy_init();
+	curl_multi_add_handle(multi_handle, file->handle.curl);
 
-  curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
-  curl_easy_setopt(file->handle.curl, CURLOPT_FILE, file);
-  curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, FALSE);
-  curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
+	/* lets start the fetch */
+	while(curl_multi_perform(multi_handle, &file->still_running) ==
+	      CURLM_CALL_MULTI_PERFORM );
 
-  if(!multi_handle)
-    multi_handle = curl_multi_init();
+	if((file->buffer_pos == 0) && (!file->still_running))
+	{
+	    /* if still_running is 0 now, we should return NULL */
 
-  curl_multi_add_handle(multi_handle, file->handle.curl);
+	    /* make sure the easy handle is not in the multi handle anymore */
+	    curl_multi_remove_handle(multi_handle, file->handle.curl);
 
-  while(CURLM_CALL_MULTI_PERFORM ==
-        curl_multi_perform(multi_handle, &still_running));
+	    /* cleanup */
+	    curl_easy_cleanup(file->handle.curl);
 
-  /* if still_running would be 0 now, we should return NULL */
+	    free(file);
 
-  return file;
+	    file = NULL;
+	}
+    }
+    return file;
 }
 
-void url_fclose(URL_FILE *file)
+int
+url_fclose(URL_FILE *file)
 {
-  /* make sure the easy handle is not in the multi handle anymore */
-  curl_multi_remove_handle(multi_handle, file->handle.curl);
+    int ret=0;/* default is good return */
 
-  /* cleanup */
-  curl_easy_cleanup(file->handle.curl);
-}
+    switch(file->type)
+    {
+    case CFTYPE_FILE:
+	ret=fclose(file->handle.file); /* passthrough */
+	break;
+
+    case CFTYPE_CURL:
+	/* make sure the easy handle is not in the multi handle anymore */
+	curl_multi_remove_handle(multi_handle, file->handle.curl);
 
+	/* cleanup */
+	curl_easy_cleanup(file->handle.curl);
+	break;
+
+    default: /* unknown or supported type - oh dear */
+	ret=EOF;
+	errno=EBADF;
+	break;
+
+    }
 
+    if(file->buffer)
+	free(file->buffer);/* free any allocated buffer space */
 
-size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
+    free(file);
+
+    return ret;
+}
+
+int
+url_feof(URL_FILE *file)
 {
-  fd_set fdread;
-  fd_set fdwrite;
-  fd_set fdexcep;
-  int maxfd;
-  struct timeval timeout;
-  int rc;
-  int still_running = 0;
-
-  if(!file->bytes) { /* no data available at this point */
-
-    file->readptr = file->buffer; /* reset read pointer */
-
-    if(CURLM_CALL_MULTI_PERFORM == file->m) {
-      while(CURLM_CALL_MULTI_PERFORM ==
-            curl_multi_perform(multi_handle, &still_running)) {
-        if(file->bytes) {
-          printf("(fread) WOAH! THis happened!\n");
-          break;        
-        }
-      }
-      if(!still_running) {
-        printf("DONE RUNNING AROUND!\n");
-        return 0;
-      }
+    int ret=0;
+
+    switch(file->type)
+    {
+    case CFTYPE_FILE:
+	ret=feof(file->handle.file);
+	break;
+
+    case CFTYPE_CURL:
+	if((file->buffer_pos == 0) && (!file->still_running))
+	    ret = 1;
+	break;
+    default: /* unknown or supported type - oh dear */
+	ret=-1;
+	errno=EBADF;
+	break;
     }
+    return ret;
+}
 
-    do {
+size_t
+url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
+{
+    size_t want;
+
+    switch(file->type)
+    {
+    case CFTYPE_FILE:
+	want=fread(ptr,size,nmemb,file->handle.file);
+	break;
+
+    case CFTYPE_CURL:
+	want = nmemb * size;
+
+	curl_fill_buffer(file,want,1);
+
+	/* check if theres data in the buffer - if not curl_fill_buffer()
+	 * either errored or EOF */
+	if(!file->buffer_pos)
+	    return 0;
+
+	/* ensure only available data is considered */
+	if(file->buffer_pos < want)
+	    want = file->buffer_pos;
 
-      FD_ZERO(&fdread);
-      FD_ZERO(&fdwrite);
-      FD_ZERO(&fdexcep);
-  
-      /* set a suitable timeout to fail on */
-      timeout.tv_sec = 500; /* 5 minutes */
-      timeout.tv_usec = 0;
-
-      /* get file descriptors from the transfers */
-      curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
-
-      rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
-
-      switch(rc) {
-      case -1:
-        /* select error */
-        break;
-      case 0:
-        break;
-      default:
-        /* timeout or readable/writable sockets */
-        printf("select() returned %d!\n", rc);
-        do {
-          file->m = curl_multi_perform(multi_handle, &still_running);
-          
-          if(file->bytes)
-            /* we have received data, return that now */
-            break;
-          
-        } while(CURLM_CALL_MULTI_PERFORM == file->m);
-
-        
-        if(!still_running)
-          printf("DONE RUNNING AROUND!\n");
-        
-        break;
-      }
-    } while(still_running && (file->bytes <= 0));
-  }
-  else
-    printf("(fread) Skip network read\n");
-
-  if(file->bytes) {
-    /* data already available, return that */
-    int want = size * nmemb;
-
-    if(file->bytes < want)
-      want = file->bytes;
-
-    memcpy(ptr, file->readptr, want);
-    file->readptr += want;
-    file->bytes -= want;
-
-    printf("(fread) return %d bytes\n", want);
+	/* xfer data to caller */
+	memcpy(ptr, file->buffer, want);
 
+	curl_use_buffer(file,want);
+
+	want = want / size;	/* number of items - nb correct op - checked
+				 * with glibc code*/
+
+	/*printf("(fread) return %d bytes %d left\n", want,file->buffer_pos);*/
+	break;
+
+    default: /* unknown or supported type - oh dear */
+	want=0;
+	errno=EBADF;
+	break;
+
+    }
     return want;
-  }
-  return 0; /* no data available to return */
 }
 
+char *
+url_fgets(char *ptr, int size, URL_FILE *file)
+{
+    int want = size - 1;/* always need to leave room for zero termination */
+    int loop;
+
+    switch(file->type)
+    {
+    case CFTYPE_FILE:
+	ptr = fgets(ptr,size,file->handle.file);
+	break;
+
+    case CFTYPE_CURL:
+	curl_fill_buffer(file,want,1);
+
+	/* check if theres data in the buffer - if not fill either errored or
+	 * EOF */
+	if(!file->buffer_pos)
+	    return NULL;
+
+	/* ensure only available data is considered */
+	if(file->buffer_pos < want)
+	    want = file->buffer_pos;
+
+	/*buffer contains data */
+	/* look for newline or eof */
+	for(loop=0;loop < want;loop++)
+	{
+	    if(file->buffer[loop] == '\n')
+	    {
+		want=loop+1;/* include newline */
+		break;
+	    }
+	}
+
+	/* xfer data to caller */
+	memcpy(ptr, file->buffer, want);
+	ptr[want]=0;/* allways null terminate */
+
+	curl_use_buffer(file,want);
+
+	/*printf("(fgets) return %d bytes %d left\n", want,file->buffer_pos);*/
+	break;
+
+    default: /* unknown or supported type - oh dear */
+	ptr=NULL;
+	errno=EBADF;
+	break;
+    }
+
+    return ptr;/*success */
+}
+
+void
+url_rewind(URL_FILE *file)
+{
+    switch(file->type)
+    {
+    case CFTYPE_FILE:
+	rewind(file->handle.file); /* passthrough */
+	break;
+
+    case CFTYPE_CURL:
+	/* halt transaction */
+	curl_multi_remove_handle(multi_handle, file->handle.curl);
+
+	/* restart */
+	curl_multi_add_handle(multi_handle, file->handle.curl);
+
+	/* ditch buffer - write will recreate - resets stream pos*/
+	if(file->buffer)
+	    free(file->buffer);
+
+	file->buffer=NULL;
+	file->buffer_pos=0;
+	file->buffer_len=0;
+
+	break;
 
-int main(int argc, char *argv[])
+    default: /* unknown or supported type - oh dear */
+	break;
+
+    }
+
+}
+
+
+/* Small main program to retrive from a url using fgets and fread saving the
+ * output to two test files (note the fgets method will corrupt binary files if
+ * they contain 0 chars */
+int
+main(int argc, char *argv[])
 {
-  URL_FILE *handle;
-  int nread;
-  char buffer[256];
+    URL_FILE *handle;
+    FILE *outf;
+
+    int nread;
+    char buffer[256];
+    char *url;
+
+    if(argc < 2)
+    {
+	url="http://192.168.7.3/testfile";/* default to testurl */
+    }
+    else
+    {
+	url=argv[1];/* use passed url */
+    }
+
+    /* copy from url line by line with fgets */
+    outf=fopen("fgets.test","w+");
+    if(!outf)
+    {
+	perror("couldnt open fgets output file\n");
+	return 1;
+    }
+
+    handle = url_fopen(url, "r");
+    if(!handle)
+    {
+	printf("couldn't url_fopen()\n");
+	fclose(outf);
+	return 2;
+    }
+
+    while(!url_feof(handle))
+    {
+	url_fgets(buffer,sizeof(buffer),handle);
+	fwrite(buffer,1,strlen(buffer),outf);
+    }
+
+    url_fclose(handle);
+
+    fclose(outf);
+
+
+    /* Copy from url with fread */
+    outf=fopen("fread.test","w+");
+    if(!outf)
+    {
+	perror("couldnt open fread output file\n");
+	return 1;
+    }
+
+    handle = url_fopen("testfile", "r");
+    if(!handle) {
+	printf("couldn't url_fopen()\n");
+	fclose(outf);
+	return 2;
+    }
+
+    do {
+	nread = url_fread(buffer, 1,sizeof(buffer), handle);
+	fwrite(buffer,1,nread,outf);
+    } while(nread);
+
+    url_fclose(handle);
+
+    fclose(outf);
+
+
+    /* Test rewind */
+    outf=fopen("rewind.test","w+");
+    if(!outf)
+    {
+	perror("couldnt open fread output file\n");
+	return 1;
+    }
+
+    handle = url_fopen("testfile", "r");
+    if(!handle) {
+	printf("couldn't url_fopen()\n");
+	fclose(outf);
+	return 2;
+    }
+
+	nread = url_fread(buffer, 1,sizeof(buffer), handle);
+	fwrite(buffer,1,nread,outf);
+	url_rewind(handle);
 
-  (void)argc;
-  (void)argv;
+	buffer[0]='\n';
+	fwrite(buffer,1,1,outf);
 
-  handle = url_fopen("http://curl.haxx.se/", "r");
+	nread = url_fread(buffer, 1,sizeof(buffer), handle);
+	fwrite(buffer,1,nread,outf);
 
-  if(!handle) {
-    printf("couldn't url_fopen()\n");
-  }
 
-  do {
-    nread = url_fread(buffer, sizeof(buffer), 1, handle);
+    url_fclose(handle);
 
-    printf("We got: %d bytes\n", nread);
-  } while(nread);
+    fclose(outf);
 
-  url_fclose(handle);
 
-  return 0;
+    return 0;/* all done */
 }