/* * fopen.c * * adapted from https://curl.haxx.se/libcurl/c/fopen.html. * * Modifications to read/write wide character streams by * Simon Brooke. * * NOTE THAT: for my purposes, I'm only interested in wide characters, * and I always read them one character at a time. * * Copyright (c) 2003, 2017 Simtec Electronics * Some portions (c) 2019 Simon Brooke * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * This example requires libcurl 7.9.7 or later. */ #include #include #ifndef WIN32 #include #endif #include #include #include #include "fopen.h" #ifdef FOPEN_STANDALONE CURLSH *io_share; #else #include "io.h" #include "consspaceobject.h" #endif /* exported functions */ URL_FILE *url_fopen( const char *url, const char *operation ); int url_fclose( URL_FILE * file ); int url_feof( URL_FILE * file ); size_t url_fread( void *ptr, size_t size, size_t nmemb, URL_FILE * file ); char *url_fgets( char *ptr, size_t size, URL_FILE * file ); void url_rewind( URL_FILE * file ); /* we use a global one for convenience */ static CURLM *multi_handle; /* curl calls this routine to get more data */ static size_t write_callback( char *buffer, size_t size, size_t nitems, void *userp ) { char *newbuff; size_t rembuff; URL_FILE *url = ( URL_FILE * ) userp; size *= nitems; rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */ if ( size > rembuff ) { /* not enough space in buffer */ newbuff = realloc( url->buffer, url->buffer_len + ( size - rembuff ) ); if ( newbuff == NULL ) { fprintf( stderr, "callback buffer grow failed\n" ); size = rembuff; } else { /* realloc succeeded increase buffer size */ url->buffer_len += size - rembuff; url->buffer = newbuff; } } memcpy( &url->buffer[url->buffer_pos], buffer, size ); url->buffer_pos += size; return size; } /* use to attempt to fill the read buffer up to requested number of bytes */ static int fill_buffer( URL_FILE * file, size_t want ) { fd_set fdread; fd_set fdwrite; fd_set fdexcep; struct timeval timeout; int rc; CURLMcode mc; /* curl_multi_fdset() return code */ /* only attempt to fill buffer if transactions still running and buffer * doesn't exceed required size already */ if ( ( !file->still_running ) || ( file->buffer_pos > want ) ) return 0; /* attempt to fill buffer */ do { int maxfd = -1; long curl_timeo = -1; FD_ZERO( &fdread ); FD_ZERO( &fdwrite ); FD_ZERO( &fdexcep ); /* set a suitable timeout to fail on */ timeout.tv_sec = 60; /* 1 minute */ timeout.tv_usec = 0; curl_multi_timeout( multi_handle, &curl_timeo ); if ( curl_timeo >= 0 ) { timeout.tv_sec = curl_timeo / 1000; if ( timeout.tv_sec > 1 ) timeout.tv_sec = 1; else timeout.tv_usec = ( curl_timeo % 1000 ) * 1000; } /* get file descriptors from the transfers */ mc = curl_multi_fdset( multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd ); if ( mc != CURLM_OK ) { fprintf( stderr, "curl_multi_fdset() failed, code %d.\n", mc ); break; } /* On success the value of maxfd is guaranteed to be >= -1. We call select(maxfd + 1, ...); specially in case of (maxfd == -1) there are no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- to sleep 100ms, which is the minimum suggested value in the curl_multi_fdset() doc. */ if ( maxfd == -1 ) { #ifdef _WIN32 Sleep( 100 ); rc = 0; #else /* Portable sleep for platforms other than Windows. */ struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ rc = select( 0, NULL, NULL, NULL, &wait ); #endif } else { /* Note that on some platforms 'timeout' may be modified by select(). If you need access to the original value save a copy beforehand. */ rc = select( maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout ); } switch ( rc ) { case -1: /* select error */ break; case 0: default: /* timeout or readable/writable sockets */ curl_multi_perform( multi_handle, &file->still_running ); break; } } while ( file->still_running && ( file->buffer_pos < want ) ); return 1; } /* use to remove want bytes from the front of a files buffer */ static int use_buffer( URL_FILE * file, size_t want ) { /* sort out buffer */ if ( ( file->buffer_pos - want ) <= 0 ) { /* ditch buffer - write will recreate */ free( file->buffer ); file->buffer = NULL; file->buffer_pos = 0; file->buffer_len = 0; } else { /* move rest down make it available for later */ memmove( file->buffer, &file->buffer[want], ( file->buffer_pos - want ) ); file->buffer_pos -= want; } return 0; } URL_FILE *url_fopen( const char *url, const char *operation ) { /* this code could check for URLs or types in the 'url' and basically use the real fopen() for standard files */ URL_FILE *file; ( void ) operation; file = calloc( 1, sizeof( URL_FILE ) ); if ( !file ) return NULL; file->handle.file = fopen( url, operation ); if ( file->handle.file ) file->type = CFTYPE_FILE; /* marked as URL */ else { file->type = CFTYPE_CURL; /* marked as URL */ file->handle.curl = curl_easy_init( ); curl_easy_setopt( file->handle.curl, CURLOPT_URL, url ); curl_easy_setopt( file->handle.curl, CURLOPT_WRITEDATA, file ); curl_easy_setopt( file->handle.curl, CURLOPT_VERBOSE, 0L ); curl_easy_setopt( file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback ); /* use the share object */ curl_easy_setopt(file->handle.curl, CURLOPT_SHARE, io_share); if ( !multi_handle ) multi_handle = curl_multi_init( ); curl_multi_add_handle( multi_handle, file->handle.curl ); /* lets start the fetch */ curl_multi_perform( multi_handle, &file->still_running ); if ( ( file->buffer_pos == 0 ) && ( !file->still_running ) ) { /* if still_running is 0 now, we should return NULL */ /* make sure the easy handle is not in the multi handle anymore */ curl_multi_remove_handle( multi_handle, file->handle.curl ); /* cleanup */ curl_easy_cleanup( file->handle.curl ); free( file ); file = NULL; } } return file; } int url_fclose( URL_FILE * file ) { int ret = 0; /* default is good return */ switch ( file->type ) { case CFTYPE_FILE: ret = fclose( file->handle.file ); /* passthrough */ break; case CFTYPE_CURL: /* make sure the easy handle is not in the multi handle anymore */ curl_multi_remove_handle( multi_handle, file->handle.curl ); /* cleanup */ curl_easy_cleanup( file->handle.curl ); break; default: /* unknown or supported type - oh dear */ ret = EOF; errno = EBADF; break; } free( file->buffer ); /* free any allocated buffer space */ free( file ); return ret; } int url_feof( URL_FILE * file ) { int ret = 0; switch ( file->type ) { case CFTYPE_FILE: ret = feof( file->handle.file ); break; case CFTYPE_CURL: if ( ( file->buffer_pos == 0 ) && ( !file->still_running ) ) ret = 1; break; default: /* unknown or supported type - oh dear */ ret = -1; errno = EBADF; break; } return ret; } size_t url_fread( void *ptr, size_t size, size_t nmemb, URL_FILE * file ) { size_t want; switch ( file->type ) { case CFTYPE_FILE: want = fread( ptr, size, nmemb, file->handle.file ); break; case CFTYPE_CURL: want = nmemb * size; fill_buffer( file, want ); /* check if there's data in the buffer - if not fill_buffer() * either errored or EOF */ if ( !file->buffer_pos ) return 0; /* ensure only available data is considered */ if ( file->buffer_pos < want ) want = file->buffer_pos; /* xfer data to caller */ memcpy( ptr, file->buffer, want ); use_buffer( file, want ); want = want / size; /* number of items */ break; default: /* unknown or supported type - oh dear */ want = 0; errno = EBADF; break; } return want; } char *url_fgets( char *ptr, size_t size, URL_FILE * file ) { size_t want = size - 1; /* always need to leave room for zero termination */ size_t loop; switch ( file->type ) { case CFTYPE_FILE: ptr = fgets( ptr, ( int ) size, file->handle.file ); break; case CFTYPE_CURL: fill_buffer( file, want ); /* check if there's data in the buffer - if not fill either errored or * EOF */ if ( !file->buffer_pos ) return NULL; /* ensure only available data is considered */ if ( file->buffer_pos < want ) want = file->buffer_pos; /*buffer contains data */ /* look for newline or eof */ for ( loop = 0; loop < want; loop++ ) { if ( file->buffer[loop] == '\n' ) { want = loop + 1; /* include newline */ break; } } /* xfer data to caller */ memcpy( ptr, file->buffer, want ); ptr[want] = 0; /* always null terminate */ use_buffer( file, want ); break; default: /* unknown or supported type - oh dear */ ptr = NULL; errno = EBADF; break; } return ptr; /*success */ } void url_rewind( URL_FILE * file ) { switch ( file->type ) { case CFTYPE_FILE: rewind( file->handle.file ); /* passthrough */ break; case CFTYPE_CURL: /* halt transaction */ curl_multi_remove_handle( multi_handle, file->handle.curl ); /* restart */ curl_multi_add_handle( multi_handle, file->handle.curl ); /* ditch buffer - write will recreate - resets stream pos */ free( file->buffer ); file->buffer = NULL; file->buffer_pos = 0; file->buffer_len = 0; break; default: /* unknown or supported type - oh dear */ break; } } #ifdef FOPEN_STANDALONE #define FGETSFILE "fgets.test" #define FREADFILE "fread.test" #define REWINDFILE "rewind.test" /* Small main program to retrieve from a url using fgets and fread saving the * output to two test files (note the fgets method will corrupt binary files if * they contain 0 chars */ int main( int argc, char *argv[] ) { URL_FILE *handle; FILE *outf; size_t nread; char buffer[256]; const char *url; CURL *curl; CURLcode res; curl_global_init( CURL_GLOBAL_DEFAULT ); curl = curl_easy_init( ); if ( argc < 2 ) url = "http://192.168.7.3/testfile"; /* default to testurl */ else url = argv[1]; /* use passed url */ /* copy from url line by line with fgets */ outf = fopen( FGETSFILE, "wb+" ); if ( !outf ) { perror( "couldn't open fgets output file\n" ); return 1; } handle = url_fopen( url, "r" ); if ( !handle ) { printf( "couldn't url_fopen() %s\n", url ); fclose( outf ); return 2; } while ( !url_feof( handle ) ) { url_fgets( buffer, sizeof( buffer ), handle ); fwrite( buffer, 1, strlen( buffer ), outf ); } url_fclose( handle ); fclose( outf ); /* Copy from url with fread */ outf = fopen( FREADFILE, "wb+" ); if ( !outf ) { perror( "couldn't open fread output file\n" ); return 1; } handle = url_fopen( "testfile", "r" ); if ( !handle ) { printf( "couldn't url_fopen() testfile\n" ); fclose( outf ); return 2; } do { nread = url_fread( buffer, 1, sizeof( buffer ), handle ); fwrite( buffer, 1, nread, outf ); } while ( nread ); url_fclose( handle ); fclose( outf ); /* Test rewind */ outf = fopen( REWINDFILE, "wb+" ); if ( !outf ) { perror( "couldn't open fread output file\n" ); return 1; } handle = url_fopen( "testfile", "r" ); if ( !handle ) { printf( "couldn't url_fopen() testfile\n" ); fclose( outf ); return 2; } nread = url_fread( buffer, 1, sizeof( buffer ), handle ); fwrite( buffer, 1, nread, outf ); url_rewind( handle ); buffer[0] = '\n'; fwrite( buffer, 1, 1, outf ); nread = url_fread( buffer, 1, sizeof( buffer ), handle ); fwrite( buffer, 1, nread, outf ); url_fclose( handle ); fclose( outf ); return 0; /* all done */ } #endif