diff options
| -rw-r--r-- | docs/examples/fopen.c | 662 |
1 files changed, 496 insertions, 166 deletions
diff --git a/docs/examples/fopen.c b/docs/examples/fopen.c index 8fc4b1f7c..452995bc3 100644 --- a/docs/examples/fopen.c +++ b/docs/examples/fopen.c @@ -1,28 +1,53 @@ /***************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| * - * $Id$ + * This example source code introduces a c library buffered I/O interface to + * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(), + * rewind(). Supported functions have identical prototypes to their normal c + * lib namesakes and are preceaded by url_ . * - * This example source code introduces an fopen()/fread()/fclose() emulation - * for URL reads. Using an approach similar to this, you could replace your - * program's fopen() with this url_fopen() and fread() with url_fread() and - * it should be possible to read remote streams instead of (only) local files. + * Using this code you can replace your program's fopen() with url_fopen() + * and fread() with url_fread() and it become possible to read remote streams + * instead of (only) local files. Local files (ie those that can be directly + * fopened) will drop back to using the underlying clib implementations * - * See the main() function at the bottom that shows a tiny app in action. + * See the main() function at the bottom that shows an app that retrives from a + * specified url using fgets() and fread() and saves as two output files. * - * This source code is a proof of concept. It will need further attention to - * become production-use useful and solid. + * Coyright (c)2003 Simtec Electronics + * + * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive + * reference to original curl example code + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * This example requires libcurl 7.9.7 or later. */ + #include <stdio.h> #include <string.h> #include <sys/time.h> #include <stdlib.h> +#include <errno.h> #include <curl/curl.h> @@ -30,206 +55,511 @@ #error "too old libcurl version, get the latest!" #endif -struct data { - int type; - union { - CURL *curl; - FILE *file; - } handle; - - /* This is the documented biggest possible buffer chunk we can get from - libcurl in one single callback! */ - char buffer[CURL_MAX_WRITE_SIZE]; - char *readptr; /* read from here */ - int bytes; /* bytes available from read pointer */ +enum fcurl_type_e { CFTYPE_NONE=0, CFTYPE_FILE=1, CFTYPE_CURL=2 }; - CURLMcode m; /* stored from a previous url_fread() */ +struct fcurl_data +{ + enum fcurl_type_e type; /* type of handle */ + union { + CURL *curl; + FILE *file; + } handle; /* handle */ + + char *buffer; /* buffer to store cached data*/ + int buffer_len; /* currently allocated buffers length */ + int buffer_pos; /* end of data in buffer*/ + int still_running; /* Is background url fetch still in progress */ }; -typedef struct data URL_FILE; +typedef struct fcurl_data URL_FILE; + +/* exported functions */ +URL_FILE *url_fopen(char *url,const char *operation); +int url_fclose(URL_FILE *file); +int url_feof(URL_FILE *file); +size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file); +char * url_fgets(char *ptr, int size, URL_FILE *file); +void url_rewind(URL_FILE *file); /* we use a global one for convenience */ CURLM *multi_handle; -static -size_t write_callback(char *buffer, - size_t size, - size_t nitems, - void *userp) +/* curl calls this routine to get more data */ +static size_t +write_callback(char *buffer, + size_t size, + size_t nitems, + void *userp) { - URL_FILE *url = (URL_FILE *)userp; - size *= nitems; + char *newbuff; + int rembuff; + + URL_FILE *url = (URL_FILE *)userp; + size *= nitems; + + rembuff=url->buffer_len - url->buffer_pos;//remaining space in buffer + + if(size > rembuff) + { + //not enuf space in buffer + newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff)); + if(newbuff==NULL) + { + fprintf(stderr,"callback buffer grow failed\n"); + size=rembuff; + } + else + { + /* realloc suceeded increase buffer size*/ + url->buffer_len+=size - rembuff; + url->buffer=newbuff; + + /*printf("Callback buffer grown to %d bytes\n",url->buffer_len);*/ + } + } + + memcpy(&url->buffer[url->buffer_pos], buffer, size); + url->buffer_pos += size; - memcpy(url->readptr, buffer, size); - url->readptr += size; - url->bytes += size; + /*fprintf(stderr, "callback %d size bytes\n", size);*/ + + return size; +} - fprintf(stderr, "callback %d size bytes\n", size); +/* use to attempt to fill the read buffer up to requested number of bytes */ +static int +curl_fill_buffer(URL_FILE *file,int want,int waittime) +{ + fd_set fdread; + fd_set fdwrite; + fd_set fdexcep; + int maxfd; + struct timeval timeout; + int rc; + + /* only attempt to fill buffer if transactions still running and buffer + * doesnt exceed required size already + */ + if((!file->still_running) || (file->buffer_pos > want)) + return 0; + + /* attempt to fill buffer */ + do + { + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcep); + + /* set a suitable timeout to fail on */ + timeout.tv_sec = 60; /* 1 minute */ + timeout.tv_usec = 0; + + /* get file descriptors from the transfers */ + curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); + + rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); + + switch(rc) { + case -1: + /* select error */ + break; + + case 0: + break; + + default: + /* timeout or readable/writable sockets */ + /* note we *could* be more efficient and not wait for + * CURLM_CALL_MULTI_PERFORM to clear here and check it on re-entry + * but that gets messy */ + while(curl_multi_perform(multi_handle, &file->still_running) == + CURLM_CALL_MULTI_PERFORM); + + break; + } + } while(file->still_running && (file->buffer_pos < want)); + return 1; +} - return size; +/* use to remove want bytes from the front of a files buffer */ +static int +curl_use_buffer(URL_FILE *file,int want) +{ + /* sort out buffer */ + if((file->buffer_pos - want) <=0) + { + /* ditch buffer - write will recreate */ + if(file->buffer) + free(file->buffer); + + file->buffer=NULL; + file->buffer_pos=0; + file->buffer_len=0; + } + else + { + /* move rest down make it available for later */ + memmove(file->buffer, + &file->buffer[want], + (file->buffer_pos - want)); + + file->buffer_pos -= want; + } + return 0; } -URL_FILE *url_fopen(char *url, char *operation) + + +URL_FILE * +url_fopen(char *url,const char *operation) { - /* this code could check for URLs or types in the 'url' and - basicly use the real fopen() for standard files */ + /* this code could check for URLs or types in the 'url' and + basicly use the real fopen() for standard files */ + + URL_FILE *file; + (void)operation; + + file = (URL_FILE *)malloc(sizeof(URL_FILE)); + if(!file) + return NULL; - URL_FILE *file; - int still_running; - (void)operation; + memset(file, 0, sizeof(URL_FILE)); + + if((file->handle.file=fopen(url,operation))) + { + file->type = CFTYPE_FILE; /* marked as URL */ + } + else + { + file->type = CFTYPE_CURL; /* marked as URL */ + file->handle.curl = curl_easy_init(); - file = (URL_FILE *)malloc(sizeof(URL_FILE)); - if(!file) - return NULL; + curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); + curl_easy_setopt(file->handle.curl, CURLOPT_FILE, file); + curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, FALSE); + curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); - memset(file, 0, sizeof(URL_FILE)); + if(!multi_handle) + multi_handle = curl_multi_init(); - file->type = 1; /* marked as URL, use 0 for plain file */ - file->handle.curl = curl_easy_init(); + curl_multi_add_handle(multi_handle, file->handle.curl); - curl_easy_setopt(file->handle.curl, CURLOPT_URL, url); - curl_easy_setopt(file->handle.curl, CURLOPT_FILE, file); - curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, FALSE); - curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback); + /* lets start the fetch */ + while(curl_multi_perform(multi_handle, &file->still_running) == + CURLM_CALL_MULTI_PERFORM ); - if(!multi_handle) - multi_handle = curl_multi_init(); + if((file->buffer_pos == 0) && (!file->still_running)) + { + /* if still_running is 0 now, we should return NULL */ - curl_multi_add_handle(multi_handle, file->handle.curl); + /* make sure the easy handle is not in the multi handle anymore */ + curl_multi_remove_handle(multi_handle, file->handle.curl); - while(CURLM_CALL_MULTI_PERFORM == - curl_multi_perform(multi_handle, &still_running)); + /* cleanup */ + curl_easy_cleanup(file->handle.curl); - /* if still_running would be 0 now, we should return NULL */ + free(file); - return file; + file = NULL; + } + } + return file; } -void url_fclose(URL_FILE *file) +int +url_fclose(URL_FILE *file) { - /* make sure the easy handle is not in the multi handle anymore */ - curl_multi_remove_handle(multi_handle, file->handle.curl); + int ret=0;/* default is good return */ - /* cleanup */ - curl_easy_cleanup(file->handle.curl); -} + switch(file->type) + { + case CFTYPE_FILE: + ret=fclose(file->handle.file); /* passthrough */ + break; + + case CFTYPE_CURL: + /* make sure the easy handle is not in the multi handle anymore */ + curl_multi_remove_handle(multi_handle, file->handle.curl); + /* cleanup */ + curl_easy_cleanup(file->handle.curl); + break; + + default: /* unknown or supported type - oh dear */ + ret=EOF; + errno=EBADF; + break; + + } + if(file->buffer) + free(file->buffer);/* free any allocated buffer space */ -size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file) + free(file); + + return ret; +} + +int +url_feof(URL_FILE *file) { - fd_set fdread; - fd_set fdwrite; - fd_set fdexcep; - int maxfd; - struct timeval timeout; - int rc; - int still_running = 0; - - if(!file->bytes) { /* no data available at this point */ - - file->readptr = file->buffer; /* reset read pointer */ - - if(CURLM_CALL_MULTI_PERFORM == file->m) { - while(CURLM_CALL_MULTI_PERFORM == - curl_multi_perform(multi_handle, &still_running)) { - if(file->bytes) { - printf("(fread) WOAH! THis happened!\n"); - break; - } - } - if(!still_running) { - printf("DONE RUNNING AROUND!\n"); - return 0; - } + int ret=0; + + switch(file->type) + { + case CFTYPE_FILE: + ret=feof(file->handle.file); + break; + + case CFTYPE_CURL: + if((file->buffer_pos == 0) && (!file->still_running)) + ret = 1; + break; + default: /* unknown or supported type - oh dear */ + ret=-1; + errno=EBADF; + break; } + return ret; +} - do { +size_t +url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file) +{ + size_t want; + + switch(file->type) + { + case CFTYPE_FILE: + want=fread(ptr,size,nmemb,file->handle.file); + break; + + case CFTYPE_CURL: + want = nmemb * size; + + curl_fill_buffer(file,want,1); + + /* check if theres data in the buffer - if not curl_fill_buffer() + * either errored or EOF */ + if(!file->buffer_pos) + return 0; + + /* ensure only available data is considered */ + if(file->buffer_pos < want) + want = file->buffer_pos; - FD_ZERO(&fdread); - FD_ZERO(&fdwrite); - FD_ZERO(&fdexcep); - - /* set a suitable timeout to fail on */ - timeout.tv_sec = 500; /* 5 minutes */ - timeout.tv_usec = 0; - - /* get file descriptors from the transfers */ - curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); - - rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); - - switch(rc) { - case -1: - /* select error */ - break; - case 0: - break; - default: - /* timeout or readable/writable sockets */ - printf("select() returned %d!\n", rc); - do { - file->m = curl_multi_perform(multi_handle, &still_running); - - if(file->bytes) - /* we have received data, return that now */ - break; - - } while(CURLM_CALL_MULTI_PERFORM == file->m); - - - if(!still_running) - printf("DONE RUNNING AROUND!\n"); - - break; - } - } while(still_running && (file->bytes <= 0)); - } - else - printf("(fread) Skip network read\n"); - - if(file->bytes) { - /* data already available, return that */ - int want = size * nmemb; - - if(file->bytes < want) - want = file->bytes; - - memcpy(ptr, file->readptr, want); - file->readptr += want; - file->bytes -= want; - - printf("(fread) return %d bytes\n", want); + /* xfer data to caller */ + memcpy(ptr, file->buffer, want); + curl_use_buffer(file,want); + + want = want / size; /* number of items - nb correct op - checked + * with glibc code*/ + + /*printf("(fread) return %d bytes %d left\n", want,file->buffer_pos);*/ + break; + + default: /* unknown or supported type - oh dear */ + want=0; + errno=EBADF; + break; + + } return want; - } - return 0; /* no data available to return */ } +char * +url_fgets(char *ptr, int size, URL_FILE *file) +{ + int want = size - 1;/* always need to leave room for zero termination */ + int loop; + + switch(file->type) + { + case CFTYPE_FILE: + ptr = fgets(ptr,size,file->handle.file); + break; + + case CFTYPE_CURL: + curl_fill_buffer(file,want,1); + + /* check if theres data in the buffer - if not fill either errored or + * EOF */ + if(!file->buffer_pos) + return NULL; + + /* ensure only available data is considered */ + if(file->buffer_pos < want) + want = file->buffer_pos; + + /*buffer contains data */ + /* look for newline or eof */ + for(loop=0;loop < want;loop++) + { + if(file->buffer[loop] == '\n') + { + want=loop+1;/* include newline */ + break; + } + } + + /* xfer data to caller */ + memcpy(ptr, file->buffer, want); + ptr[want]=0;/* allways null terminate */ + + curl_use_buffer(file,want); + + /*printf("(fgets) return %d bytes %d left\n", want,file->buffer_pos);*/ + break; + + default: /* unknown or supported type - oh dear */ + ptr=NULL; + errno=EBADF; + break; + } + + return ptr;/*success */ +} + +void +url_rewind(URL_FILE *file) +{ + switch(file->type) + { + case CFTYPE_FILE: + rewind(file->handle.file); /* passthrough */ + break; + + case CFTYPE_CURL: + /* halt transaction */ + curl_multi_remove_handle(multi_handle, file->handle.curl); + + /* restart */ + curl_multi_add_handle(multi_handle, file->handle.curl); + + /* ditch buffer - write will recreate - resets stream pos*/ + if(file->buffer) + free(file->buffer); + + file->buffer=NULL; + file->buffer_pos=0; + file->buffer_len=0; + + break; -int main(int argc, char *argv[]) + default: /* unknown or supported type - oh dear */ + break; + + } + +} + + +/* Small main program to retrive from a url using fgets and fread saving the + * output to two test files (note the fgets method will corrupt binary files if + * they contain 0 chars */ +int +main(int argc, char *argv[]) { - URL_FILE *handle; - int nread; - char buffer[256]; + URL_FILE *handle; + FILE *outf; + + int nread; + char buffer[256]; + char *url; + + if(argc < 2) + { + url="http://192.168.7.3/testfile";/* default to testurl */ + } + else + { + url=argv[1];/* use passed url */ + } + + /* copy from url line by line with fgets */ + outf=fopen("fgets.test","w+"); + if(!outf) + { + perror("couldnt open fgets output file\n"); + return 1; + } + + handle = url_fopen(url, "r"); + if(!handle) + { + printf("couldn't url_fopen()\n"); + fclose(outf); + return 2; + } + + while(!url_feof(handle)) + { + url_fgets(buffer,sizeof(buffer),handle); + fwrite(buffer,1,strlen(buffer),outf); + } + + url_fclose(handle); + + fclose(outf); + + + /* Copy from url with fread */ + outf=fopen("fread.test","w+"); + if(!outf) + { + perror("couldnt open fread output file\n"); + return 1; + } + + handle = url_fopen("testfile", "r"); + if(!handle) { + printf("couldn't url_fopen()\n"); + fclose(outf); + return 2; + } + + do { + nread = url_fread(buffer, 1,sizeof(buffer), handle); + fwrite(buffer,1,nread,outf); + } while(nread); + + url_fclose(handle); + + fclose(outf); + + + /* Test rewind */ + outf=fopen("rewind.test","w+"); + if(!outf) + { + perror("couldnt open fread output file\n"); + return 1; + } + + handle = url_fopen("testfile", "r"); + if(!handle) { + printf("couldn't url_fopen()\n"); + fclose(outf); + return 2; + } + + nread = url_fread(buffer, 1,sizeof(buffer), handle); + fwrite(buffer,1,nread,outf); + url_rewind(handle); - (void)argc; - (void)argv; + buffer[0]='\n'; + fwrite(buffer,1,1,outf); - handle = url_fopen("http://curl.haxx.se/", "r"); + nread = url_fread(buffer, 1,sizeof(buffer), handle); + fwrite(buffer,1,nread,outf); - if(!handle) { - printf("couldn't url_fopen()\n"); - } - do { - nread = url_fread(buffer, sizeof(buffer), 1, handle); + url_fclose(handle); - printf("We got: %d bytes\n", nread); - } while(nread); + fclose(outf); - url_fclose(handle); - return 0; + return 0;/* all done */ } |