simon/post-scarcity
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

It works!

Browse source
This commit is contained in:
Simon Brooke 2019-01-28 18:46:24 +00:00
parent b15c0e8f89
commit a640c9dff9
10 changed files with 568 additions and 438 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -34,3 +34,5 @@ utils_src/readprintwc/out
*.dump *.dump
*.bak *.bak
src/io/fopen

1
lisp/slurp.lisp Normal file
View file

@ -0,0 +1 @@
(slurp (set! f (open "http://www.journeyman.cc/")))

2
src/debug.c
View file

@ -19,9 +19,9 @@
#include <wctype.h> #include <wctype.h>
#include "consspaceobject.h" #include "consspaceobject.h"
#include "fopen.h"
#include "debug.h" #include "debug.h"
#include "dump.h" #include "dump.h"
#include "io.h"
#include "print.h" #include "print.h"
/** /**

855
src/io/fopen.c
View file

@ -37,517 +37,510 @@
* This example requires libcurl 7.9.7 or later. * This example requires libcurl 7.9.7 or later.
*/ */
#include <errno.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h>
#include <string.h> #include <string.h>
#ifndef WIN32 #ifndef WIN32
#include <sys/time.h> #include <sys/time.h>
#endif #endif
#include <stdlib.h>
#include <errno.h>
#include <curl/curl.h> #include <curl/curl.h>
#include "debug.h" enum fcurl_type_e {
#include "fopen.h" CFTYPE_NONE = 0,
CFTYPE_FILE = 1,
CFTYPE_CURL = 2
};
struct fcurl_data
{
enum fcurl_type_e type; /* type of handle */
union {
CURL *curl;
FILE *file;
} handle; /* handle */
char *buffer; /* buffer to store cached data*/
size_t buffer_len; /* currently allocated buffers length */
size_t buffer_pos; /* end of data in buffer*/
int still_running; /* Is background url fetch still in progress */
};
typedef struct fcurl_data URL_FILE;
/* exported functions */
URL_FILE *url_fopen(const char *url, const char *operation);
int url_fclose(URL_FILE *file);
int url_feof(URL_FILE *file);
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
char *url_fgets(char *ptr, size_t size, URL_FILE *file);
void url_rewind(URL_FILE *file);
/* we use a global one for convenience */ /* we use a global one for convenience */
static CURLM *multi_handle; static CURLM *multi_handle;
wint_t ungotten = 0;
/* curl calls this routine to get more data */ /* curl calls this routine to get more data */
static size_t write_callback( char *buffer, static size_t write_callback(char *buffer,
size_t size, size_t nitems, void *userp ) { size_t size,
char *newbuff; size_t nitems,
size_t rembuff; void *userp)
{
char *newbuff;
size_t rembuff;
URL_FILE *url = ( URL_FILE * ) userp; URL_FILE *url = (URL_FILE *)userp;
size *= nitems; size *= nitems;
rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */ rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */
if ( size > rembuff ) { if(size > rembuff) {
/* not enough space in buffer */ /* not enough space in buffer */
newbuff = realloc( url->buffer, url->buffer_len + ( size - rembuff ) ); newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff));
if ( newbuff == NULL ) { if(newbuff == NULL) {
fprintf( stderr, "callback buffer grow failed\n" ); fprintf(stderr, "callback buffer grow failed\n");
size = rembuff; size = rembuff;
} else {
/* realloc succeeded increase buffer size */
url->buffer_len += size - rembuff;
url->buffer = newbuff;
}
} }
else {
/* realloc succeeded increase buffer size*/
url->buffer_len += size - rembuff;
url->buffer = newbuff;
}
}
memcpy( &url->buffer[url->buffer_pos], buffer, size ); memcpy(&url->buffer[url->buffer_pos], buffer, size);
url->buffer_pos += size; url->buffer_pos += size;
return size; return size;
} }
/* use to attempt to fill the read buffer up to requested number of bytes */ /* use to attempt to fill the read buffer up to requested number of bytes */
static int fill_buffer( URL_FILE * file, size_t want ) { static int fill_buffer(URL_FILE *file, size_t want)
fd_set fdread; {
fd_set fdwrite; fd_set fdread;
fd_set fdexcep; fd_set fdwrite;
struct timeval timeout; fd_set fdexcep;
int rc; struct timeval timeout;
CURLMcode mc; /* curl_multi_fdset() return code */ int rc;
CURLMcode mc; /* curl_multi_fdset() return code */
/* only attempt to fill buffer if transactions still running and buffer /* only attempt to fill buffer if transactions still running and buffer
* doesn't exceed required size already * doesn't exceed required size already
*/ */
if ( ( !file->still_running ) || ( file->buffer_pos > want ) ) if((!file->still_running) || (file->buffer_pos > want))
return 0; return 0;
/* attempt to fill buffer */ /* attempt to fill buffer */
do { do {
int maxfd = -1; int maxfd = -1;
long curl_timeo = -1; long curl_timeo = -1;
FD_ZERO( &fdread ); FD_ZERO(&fdread);
FD_ZERO( &fdwrite ); FD_ZERO(&fdwrite);
FD_ZERO( &fdexcep ); FD_ZERO(&fdexcep);
/* set a suitable timeout to fail on */ /* set a suitable timeout to fail on */
timeout.tv_sec = 60; /* 1 minute */ timeout.tv_sec = 60; /* 1 minute */
timeout.tv_usec = 0; timeout.tv_usec = 0;
curl_multi_timeout( multi_handle, &curl_timeo ); curl_multi_timeout(multi_handle, &curl_timeo);
if ( curl_timeo >= 0 ) { if(curl_timeo >= 0) {
timeout.tv_sec = curl_timeo / 1000; timeout.tv_sec = curl_timeo / 1000;
if ( timeout.tv_sec > 1 ) if(timeout.tv_sec > 1)
timeout.tv_sec = 1; timeout.tv_sec = 1;
else else
timeout.tv_usec = ( curl_timeo % 1000 ) * 1000; timeout.tv_usec = (curl_timeo % 1000) * 1000;
} }
/* get file descriptors from the transfers */ /* get file descriptors from the transfers */
mc = curl_multi_fdset( multi_handle, &fdread, &fdwrite, &fdexcep, mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
&maxfd );
if ( mc != CURLM_OK ) { if(mc != CURLM_OK) {
fprintf( stderr, "curl_multi_fdset() failed, code %d.\n", mc ); fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
break; break;
} }
/* On success the value of maxfd is guaranteed to be >= -1. We call /* On success the value of maxfd is guaranteed to be >= -1. We call
select(maxfd + 1, ...); specially in case of (maxfd == -1) there are select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
no fds ready yet so we call select(0, ...) --or Sleep() on Windows-- no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
to sleep 100ms, which is the minimum suggested value in the to sleep 100ms, which is the minimum suggested value in the
curl_multi_fdset() doc. */ curl_multi_fdset() doc. */
if ( maxfd == -1 ) { if(maxfd == -1) {
#ifdef _WIN32 #ifdef _WIN32
Sleep( 100 ); Sleep(100);
rc = 0; rc = 0;
#else #else
/* Portable sleep for platforms other than Windows. */ /* Portable sleep for platforms other than Windows. */
struct timeval wait = { 0, 100 * 1000 }; /* 100ms */ struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
rc = select( 0, NULL, NULL, NULL, &wait ); rc = select(0, NULL, NULL, NULL, &wait);
#endif #endif
} else { }
/* Note that on some platforms 'timeout' may be modified by select(). else {
If you need access to the original value save a copy beforehand. */ /* Note that on some platforms 'timeout' may be modified by select().
rc = select( maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout ); If you need access to the original value save a copy beforehand. */
} rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
}
switch ( rc ) { switch(rc) {
case -1: case -1:
/* select error */ /* select error */
break; break;
case 0: case 0:
default: default:
/* timeout or readable/writable sockets */ /* timeout or readable/writable sockets */
curl_multi_perform( multi_handle, &file->still_running ); curl_multi_perform(multi_handle, &file->still_running);
break; break;
} }
} while ( file->still_running && ( file->buffer_pos < want ) ); } while(file->still_running && (file->buffer_pos < want));
return 1;
return 1;
} }
/* use to remove want bytes from the front of a files buffer */ /* use to remove want bytes from the front of a files buffer */
static int use_buffer( URL_FILE * file, size_t want ) { static int use_buffer(URL_FILE *file, size_t want)
/* sort out buffer */ {
if ( ( file->buffer_pos - want ) <= 0 ) { /* sort out buffer */
/* ditch buffer - write will recreate */ if((file->buffer_pos - want) <= 0) {
free( file->buffer ); /* ditch buffer - write will recreate */
file->buffer = NULL; free(file->buffer);
file->buffer_pos = 0; file->buffer = NULL;
file->buffer_len = 0; file->buffer_pos = 0;
} else { file->buffer_len = 0;
/* move rest down make it available for later */ }
memmove( file->buffer, else {
&file->buffer[want], ( file->buffer_pos - want ) ); /* move rest down make it available for later */
memmove(file->buffer,
&file->buffer[want],
(file->buffer_pos - want));
file->buffer_pos -= want; file->buffer_pos -= want;
} }
return 0; return 0;
} }
/** URL_FILE *url_fopen(const char *url, const char *operation)
* consume one wide character on the buffer of this file. {
* /* this code could check for URLs or types in the 'url' and
* @param file the url or file from which the character is consumed. basically use the real fopen() for standard files */
*/
static int use_one_wide( URL_FILE * file ) {
int c = ( int ) file->buffer[file->buffer_pos];
size_t count = 0;
/* The value of each individual byte indicates its UTF-8 function, as follows: URL_FILE *file;
* (void)operation;
* 00 to 7F hex (0 to 127): first and only byte of a sequence.
* 80 to BF hex (128 to 191): continuing byte in a multi-byte sequence. file = calloc(1, sizeof(URL_FILE));
* C2 to DF hex (194 to 223): first byte of a two-byte sequence. if(!file)
* E0 to EF hex (224 to 239): first byte of a three-byte sequence. return NULL;
* F0 to FF hex (240 to 255): first byte of a four-byte sequence.
*/ file->handle.file = fopen(url, operation);
if ( c <= '0x07' ) { if(file->handle.file)
count = 1; file->type = CFTYPE_FILE; /* marked as URL */
} else if ( c >= '0xc2' && c <= '0xdf' ) {
count = 2; else {
} else if ( c >= '0xe0' && c <= '0xef' ) { file->type = CFTYPE_CURL; /* marked as URL */
count = 3; file->handle.curl = curl_easy_init();
} else if ( c >= '0xf0' && c <= '0xff' ) {
count = 4; curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
if(!multi_handle)
multi_handle = curl_multi_init();
curl_multi_add_handle(multi_handle, file->handle.curl);
/* lets start the fetch */
curl_multi_perform(multi_handle, &file->still_running);
if((file->buffer_pos == 0) && (!file->still_running)) {
/* if still_running is 0 now, we should return NULL */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* cleanup */
curl_easy_cleanup(file->handle.curl);
free(file);
file = NULL;
} }
}
return use_buffer( file, c ); return file;
} }
URL_FILE *url_fopen( const char *url, const char *operation ) { int url_fclose(URL_FILE *file)
/* this code could check for URLs or types in the 'url' and {
basically use the real fopen() for standard files */ int ret = 0;/* default is good return */
URL_FILE *file; switch(file->type) {
( void ) operation; case CFTYPE_FILE:
ret = fclose(file->handle.file); /* passthrough */
break;
file = calloc( 1, sizeof( URL_FILE ) ); case CFTYPE_CURL:
if ( !file ) /* make sure the easy handle is not in the multi handle anymore */
return NULL; curl_multi_remove_handle(multi_handle, file->handle.curl);
file->handle.file = fopen( url, operation ); /* cleanup */
if ( file->handle.file ) curl_easy_cleanup(file->handle.curl);
file->type = CFTYPE_FILE; /* marked as URL */ break;
else { default: /* unknown or supported type - oh dear */
file->type = CFTYPE_CURL; /* marked as URL */ ret = EOF;
file->handle.curl = curl_easy_init( ); errno = EBADF;
break;
}
curl_easy_setopt( file->handle.curl, CURLOPT_URL, url ); free(file->buffer);/* free any allocated buffer space */
curl_easy_setopt( file->handle.curl, CURLOPT_WRITEDATA, file ); free(file);
curl_easy_setopt( file->handle.curl, CURLOPT_VERBOSE, 0L );
curl_easy_setopt( file->handle.curl, CURLOPT_WRITEFUNCTION,
write_callback );
if ( !multi_handle ) return ret;
multi_handle = curl_multi_init( );
curl_multi_add_handle( multi_handle, file->handle.curl );
/* lets start the fetch */
curl_multi_perform( multi_handle, &file->still_running );
if ( ( file->buffer_pos == 0 ) && ( !file->still_running ) ) {
/* if still_running is 0 now, we should return NULL */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle( multi_handle, file->handle.curl );
/* cleanup */
curl_easy_cleanup( file->handle.curl );
free( file );
file = NULL;
}
}
return file;
} }
int url_fclose( URL_FILE * file ) { int url_feof(URL_FILE *file)
int ret = 0; /* default is good return */ {
int ret = 0;
switch ( file->type ) { switch(file->type) {
case CFTYPE_FILE: case CFTYPE_FILE:
ret = fclose( file->handle.file ); /* passthrough */ ret = feof(file->handle.file);
break; break;
case CFTYPE_CURL: case CFTYPE_CURL:
/* make sure the easy handle is not in the multi handle anymore */ if((file->buffer_pos == 0) && (!file->still_running))
curl_multi_remove_handle( multi_handle, file->handle.curl ); ret = 1;
break;
/* cleanup */ default: /* unknown or supported type - oh dear */
curl_easy_cleanup( file->handle.curl ); ret = -1;
break; errno = EBADF;
break;
default: /* unknown or supported type - oh dear */ }
ret = EOF; return ret;
errno = EBADF;
break;
}
free( file->buffer ); /* free any allocated buffer space */
free( file );
return ret;
} }
int url_feof( URL_FILE * file ) { size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
int ret = 0; {
size_t want;
switch ( file->type ) { switch(file->type) {
case CFTYPE_FILE: case CFTYPE_FILE:
ret = feof( file->handle.file ); want = fread(ptr, size, nmemb, file->handle.file);
break; break;
case CFTYPE_CURL: case CFTYPE_CURL:
if ( ( file->buffer_pos == 0 ) && ( !file->still_running ) ) want = nmemb * size;
ret = 1;
break;
default: /* unknown or supported type - oh dear */ fill_buffer(file, want);
ret = -1;
errno = EBADF; /* check if there's data in the buffer - if not fill_buffer()
break; * either errored or EOF */
} if(!file->buffer_pos)
return ret; return 0;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
use_buffer(file, want);
want = want / size; /* number of items */
break;
default: /* unknown or supported type - oh dear */
want = 0;
errno = EBADF;
break;
}
return want;
} }
size_t url_fread( void *ptr, size_t size, size_t nmemb, URL_FILE * file ) { char *url_fgets(char *ptr, size_t size, URL_FILE *file)
size_t want; {
size_t want = size - 1;/* always need to leave room for zero termination */
size_t loop;
switch ( file->type ) { switch(file->type) {
case CFTYPE_FILE: case CFTYPE_FILE:
want = fread( ptr, size, nmemb, file->handle.file ); ptr = fgets(ptr, (int)size, file->handle.file);
break; break;
case CFTYPE_CURL: case CFTYPE_CURL:
want = nmemb * size; fill_buffer(file, want);
fill_buffer( file, want ); /* check if there's data in the buffer - if not fill either errored or
* EOF */
if(!file->buffer_pos)
return NULL;
/* check if there's data in the buffer - if not fill_buffer() /* ensure only available data is considered */
* either errored or EOF */ if(file->buffer_pos < want)
if ( !file->buffer_pos ) want = file->buffer_pos;
return 0;
/* ensure only available data is considered */ /*buffer contains data */
if ( file->buffer_pos < want ) /* look for newline or eof */
want = file->buffer_pos; for(loop = 0; loop < want; loop++) {
if(file->buffer[loop] == '\n') {
/* xfer data to caller */ want = loop + 1;/* include newline */
memcpy( ptr, file->buffer, want ); break;
}
use_buffer( file, want );
want = want / size; /* number of items */
break;
default: /* unknown or supported type - oh dear */
want = 0;
errno = EBADF;
break;
}
return want;
}
char *url_fgets( char *ptr, size_t size, URL_FILE * file ) {
size_t want = size - 1; /* always need to leave room for zero termination */
size_t loop;
switch ( file->type ) {
case CFTYPE_FILE:
ptr = fgets( ptr, ( int ) size, file->handle.file );
break;
case CFTYPE_CURL:
fill_buffer( file, want );
/* check if there's data in the buffer - if not fill either errored or
* EOF */
if ( !file->buffer_pos )
return NULL;
/* ensure only available data is considered */
if ( file->buffer_pos < want )
want = file->buffer_pos;
/*buffer contains data */
/* look for newline or eof */
for ( loop = 0; loop < want; loop++ ) {
if ( file->buffer[loop] == '\n' ) {
want = loop + 1; /* include newline */
break;
}
}
/* xfer data to caller */
memcpy( ptr, file->buffer, want );
ptr[want] = 0; /* always null terminate */
use_buffer( file, want );
break;
default: /* unknown or supported type - oh dear */
ptr = NULL;
errno = EBADF;
break;
} }
return ptr; /*success */ /* xfer data to caller */
memcpy(ptr, file->buffer, want);
ptr[want] = 0;/* always null terminate */
use_buffer(file, want);
break;
default: /* unknown or supported type - oh dear */
ptr = NULL;
errno = EBADF;
break;
}
return ptr;/*success */
} }
void url_rewind( URL_FILE * file ) { void url_rewind(URL_FILE *file)
switch ( file->type ) { {
case CFTYPE_FILE: switch(file->type) {
rewind( file->handle.file ); /* passthrough */ case CFTYPE_FILE:
break; rewind(file->handle.file); /* passthrough */
break;
case CFTYPE_CURL: case CFTYPE_CURL:
/* halt transaction */ /* halt transaction */
curl_multi_remove_handle( multi_handle, file->handle.curl ); curl_multi_remove_handle(multi_handle, file->handle.curl);
/* restart */ /* restart */
curl_multi_add_handle( multi_handle, file->handle.curl ); curl_multi_add_handle(multi_handle, file->handle.curl);
/* ditch buffer - write will recreate - resets stream pos */ /* ditch buffer - write will recreate - resets stream pos*/
free( file->buffer ); free(file->buffer);
file->buffer = NULL; file->buffer = NULL;
file->buffer_pos = 0; file->buffer_pos = 0;
file->buffer_len = 0; file->buffer_len = 0;
break; break;
default: /* unknown or supported type - oh dear */ default: /* unknown or supported type - oh dear */
break; break;
} }
} }
/** #ifdef FOPEN_STANDALONE
* given this file handle f, return a new url_file handle wrapping it. #define FGETSFILE "fgets.test"
* #define FREADFILE "fread.test"
* @param f the file to be wrapped; #define REWINDFILE "rewind.test"
* @return the new handle, or null if no such handle could be allocated.
*/
URL_FILE *file_to_url_file( FILE * f ) {
URL_FILE *result = ( URL_FILE * ) malloc( sizeof( URL_FILE ) );
if ( result != NULL ) { /* Small main program to retrieve from a url using fgets and fread saving the
result->type = CFTYPE_FILE, result->handle.file = f; * output to two test files (note the fgets method will corrupt binary files if
} * they contain 0 chars */
int main(int argc, char *argv[])
{
URL_FILE *handle;
FILE *outf;
return result; size_t nread;
} char buffer[256];
const char *url;
/** CURL *curl;
* get one wide character from the buffer. CURLcode res;
*
* @param file the stream to read from; curl_global_init(CURL_GLOBAL_DEFAULT);
* @return the next wide character on the stream, or zero if no more.
*/ curl = curl_easy_init();
wint_t url_fgetwc( URL_FILE * input ) {
wint_t result = -1;
if(argc < 2)
debug_printf( DEBUG_IO, L"url_fgetwc: ungotten = %d\n", ungotten ); url = "http://192.168.7.3/testfile";/* default to testurl */
else
if ( ungotten != 0 ) { url = argv[1];/* use passed url */
/* TODO: not thread safe */
result = ungotten; /* copy from url line by line with fgets */
ungotten = 0; outf = fopen(FGETSFILE, "wb+");
} else { if(!outf) {
switch ( input->type ) { perror("couldn't open fgets output file\n");
case CFTYPE_FILE: return 1;
fwide( input->handle.file, 1 ); /* wide characters */ }
result = fgetwc( input->handle.file ); /* passthrough */
break; handle = url_fopen(url, "r");
if(!handle) {
case CFTYPE_CURL:{ printf("couldn't url_fopen() %s\n", url);
debug_print( L"url_fgetwc: stream is URL\n", DEBUG_IO ); fclose(outf);
return 2;
char *cbuff = }
calloc( sizeof( wchar_t ) + 1, sizeof( char ) );
wchar_t *wbuff = calloc( 2, sizeof( wchar_t ) ); while(!url_feof(handle)) {
url_fgets(buffer, sizeof(buffer), handle);
size_t count = 0; fwrite(buffer, 1, strlen(buffer), outf);
}
debug_print( L"url_fgetwc: about to call url_fgets\n", DEBUG_IO );
url_fgets( cbuff, 1, input ); url_fclose(handle);
debug_print( L"url_fgetwc: back from url_fgets\n", DEBUG_IO );
int c = ( int ) cbuff[0]; fclose(outf);
debug_printf( DEBUG_IO, L"url_fgetwc: (first) character = %d (%c)\n", c, c & 0xf7 );
/* The value of each individual byte indicates its UTF-8 function, as follows:
* /* Copy from url with fread */
* 00 to 7F hex (0 to 127): first and only byte of a sequence. outf = fopen(FREADFILE, "wb+");
* 80 to BF hex (128 to 191): continuing byte in a multi-byte sequence. if(!outf) {
* C2 to DF hex (194 to 223): first byte of a two-byte sequence. perror("couldn't open fread output file\n");
* E0 to EF hex (224 to 239): first byte of a three-byte sequence. return 1;
* F0 to FF hex (240 to 255): first byte of a four-byte sequence. }
*/
if ( c <= 0x07 ) { handle = url_fopen("testfile", "r");
count = 1; if(!handle) {
} else if ( c >= '0xc2' && c <= '0xdf' ) { printf("couldn't url_fopen() testfile\n");
count = 2; fclose(outf);
} else if ( c >= '0xe0' && c <= '0xef' ) { return 2;
count = 3; }
} else if ( c >= '0xf0' && c <= '0xff' ) {
count = 4; do {
} nread = url_fread(buffer, 1, sizeof(buffer), handle);
fwrite(buffer, 1, nread, outf);
if ( count > 1 ) { } while(nread);
url_fgets( cbuff, --count, input );
} url_fclose(handle);
mbstowcs( wbuff, cbuff, 1 ); //(char *)(&input->buffer[input->buffer_pos]), 1 );
result = wbuff[0]; fclose(outf);
use_one_wide( input );
free( wbuff ); /* Test rewind */
free( cbuff ); outf = fopen(REWINDFILE, "wb+");
} if(!outf) {
break; perror("couldn't open fread output file\n");
case CFTYPE_NONE: return 1;
break; }
}
} handle = url_fopen("testfile", "r");
if(!handle) {
debug_printf( DEBUG_IO, L"url_fgetwc returning %d (%C)\n", result, printf("couldn't url_fopen() testfile\n");
result ); fclose(outf);
return result; return 2;
} }
wint_t url_ungetwc( wint_t wc, URL_FILE * input ) { nread = url_fread(buffer, 1, sizeof(buffer), handle);
wint_t result = -1; fwrite(buffer, 1, nread, outf);
url_rewind(handle);
switch ( input->type ) {
case CFTYPE_FILE: buffer[0]='\n';
fwide( input->handle.file, 1 ); /* wide characters */ fwrite(buffer, 1, 1, outf);
result = ungetwc( wc, input->handle.file ); /* passthrough */
break; nread = url_fread(buffer, 1, sizeof(buffer), handle);
fwrite(buffer, 1, nread, outf);
case CFTYPE_CURL:{
ungotten = wc; url_fclose(handle);
// wchar_t *wbuff = calloc( 2, sizeof( wchar_t ) );
// char *cbuff = calloc( 5, sizeof( char ) ); fclose(outf);
//
// wbuff[0] = wc; return 0;/* all done */
// result = wcstombs( cbuff, wbuff, 1 );
//
// input->buffer_pos -= strlen( cbuff );
//
// free( cbuff );
// free( wbuff );
//
// result = result > 0 ? wc : result;
break;
case CFTYPE_NONE:
break;
}
}
return result;
} }
#endif

4
src/io/fopen.h
View file

@ -80,8 +80,4 @@ size_t url_fread( void *ptr, size_t size, size_t nmemb, URL_FILE * file );
char *url_fgets( char *ptr, size_t size, URL_FILE * file ); char *url_fgets( char *ptr, size_t size, URL_FILE * file );
void url_rewind( URL_FILE * file ); void url_rewind( URL_FILE * file );
wint_t url_fgetwc( URL_FILE * file );
wint_t url_ungetwc( wint_t wc, URL_FILE * input );
URL_FILE *file_to_url_file( FILE * f );
#endif #endif

131
src/io/io.c
View file

@ -15,6 +15,12 @@
#include "fopen.h" #include "fopen.h"
#include "lispops.h" #include "lispops.h"
/**
* Allow a one-character unget facility. This may not be enough - we may need
* to allocate a buffer.
*/
wint_t ungotten = 0;
/** /**
* Convert this lisp string-like-thing (also works for symbols, and, later * Convert this lisp string-like-thing (also works for symbols, and, later
* keywords) into a UTF-8 string. NOTE that the returned value has been * keywords) into a UTF-8 string. NOTE that the returned value has been
@ -56,6 +62,129 @@ char *lisp_string_to_c_string( struct cons_pointer s ) {
return result; return result;
} }
/**
* given this file handle f, return a new url_file handle wrapping it.
*
* @param f the file to be wrapped;
* @return the new handle, or null if no such handle could be allocated.
*/
URL_FILE *file_to_url_file( FILE * f ) {
URL_FILE *result = ( URL_FILE * ) malloc( sizeof( URL_FILE ) );
if ( result != NULL ) {
result->type = CFTYPE_FILE, result->handle.file = f;
}
return result;
}
/**
* get one wide character from the buffer.
*
* @param file the stream to read from;
* @return the next wide character on the stream, or zero if no more.
*/
wint_t url_fgetwc( URL_FILE * input ) {
wint_t result = -1;
if ( ungotten != 0 ) {
/* TODO: not thread safe */
result = ungotten;
ungotten = 0;
} else {
switch ( input->type ) {
case CFTYPE_FILE:
fwide( input->handle.file, 1 ); /* wide characters */
result = fgetwc( input->handle.file ); /* passthrough */
break;
case CFTYPE_CURL:{
char *cbuff =
calloc( sizeof( wchar_t ) + 2, sizeof( char ) );
wchar_t *wbuff = calloc( 2, sizeof( wchar_t ) );
size_t count = 0;
debug_print( L"url_fgetwc: about to call url_fgets\n", DEBUG_IO );
url_fgets( cbuff, 2, input );
debug_print( L"url_fgetwc: back from url_fgets\n", DEBUG_IO );
int c = ( int ) cbuff[0];
debug_printf( DEBUG_IO,
L"url_fgetwc: cbuff is '%s'; (first) character = %d (%c)\n",
cbuff, c, c & 0xf7 );
/* The value of each individual byte indicates its UTF-8 function, as follows:
*
* 00 to 7F hex (0 to 127): first and only byte of a sequence.
* 80 to BF hex (128 to 191): continuing byte in a multi-byte sequence.
* C2 to DF hex (194 to 223): first byte of a two-byte sequence.
* E0 to EF hex (224 to 239): first byte of a three-byte sequence.
* F0 to FF hex (240 to 255): first byte of a four-byte sequence.
*/
if ( c <= 0x07 ) {
count = 1;
} else if ( c >= '0xc2' && c <= '0xdf' ) {
count = 2;
} else if ( c >= '0xe0' && c <= '0xef' ) {
count = 3;
} else if ( c >= '0xf0' && c <= '0xff' ) {
count = 4;
}
if ( count > 1 ) {
url_fgets( (char *)&cbuff[1], count, input );
}
mbstowcs( wbuff, cbuff, 1 ); //(char *)(&input->buffer[input->buffer_pos]), 1 );
result = wbuff[0];
free( wbuff );
free( cbuff );
}
break;
case CFTYPE_NONE:
break;
}
}
debug_printf( DEBUG_IO, L"url_fgetwc returning %d (%C)\n", result,
result );
return result;
}
wint_t url_ungetwc( wint_t wc, URL_FILE * input ) {
wint_t result = -1;
switch ( input->type ) {
case CFTYPE_FILE:
fwide( input->handle.file, 1 ); /* wide characters */
result = ungetwc( wc, input->handle.file ); /* passthrough */
break;
case CFTYPE_CURL:{
ungotten = wc;
// wchar_t *wbuff = calloc( 2, sizeof( wchar_t ) );
// char *cbuff = calloc( 5, sizeof( char ) );
//
// wbuff[0] = wc;
// result = wcstombs( cbuff, wbuff, 1 );
//
// input->buffer_pos -= strlen( cbuff );
//
// free( cbuff );
// free( wbuff );
//
// result = result > 0 ? wc : result;
break;
case CFTYPE_NONE:
break;
}
}
return result;
}
/** /**
* Function, sort-of: close the file indicated by my first arg, and return * Function, sort-of: close the file indicated by my first arg, and return
* nil. If the first arg is not a stream, does nothing. All other args are * nil. If the first arg is not a stream, does nothing. All other args are
@ -172,7 +301,7 @@ lisp_slurp( struct stack_frame *frame, struct cons_pointer frame_pointer,
struct cons_pointer cursor = make_string( url_fgetwc( stream ), NIL ); struct cons_pointer cursor = make_string( url_fgetwc( stream ), NIL );
result = cursor; result = cursor;
for ( wint_t c = url_fgetwc( stream ); !url_feof( stream ); for ( wint_t c = url_fgetwc( stream ); !url_feof( stream ) && c != 0;
c = url_fgetwc( stream ) ) { c = url_fgetwc( stream ) ) {
debug_print( L"slurp: cursor is: ", DEBUG_IO ); debug_print( L"slurp: cursor is: ", DEBUG_IO );
debug_dump_object( cursor, DEBUG_IO ); debug_dump_object( cursor, DEBUG_IO );

4
src/io/io.h
View file

@ -11,6 +11,10 @@
#ifndef __psse_io_h #ifndef __psse_io_h
#define __psse_io_h #define __psse_io_h
URL_FILE *file_to_url_file( FILE * f );
wint_t url_fgetwc( URL_FILE * input );
wint_t url_ungetwc( wint_t wc, URL_FILE * input );
struct cons_pointer struct cons_pointer
lisp_close( struct stack_frame *frame, struct cons_pointer frame_pointer, lisp_close( struct stack_frame *frame, struct cons_pointer frame_pointer,
struct cons_pointer env ); struct cons_pointer env );

4
src/memory/conspage.c
View file

@ -166,6 +166,10 @@ void free_cell( struct cons_pointer pointer ) {
dec_ref( cell->payload.ratio.dividend ); dec_ref( cell->payload.ratio.dividend );
dec_ref( cell->payload.ratio.divisor ); dec_ref( cell->payload.ratio.divisor );
break; break;
case READTV:
case WRITETV:
url_fclose( cell->payload.stream.stream);
break;
case SPECIALTV: case SPECIALTV:
dec_ref( cell->payload.special.source ); dec_ref( cell->payload.special.source );
break; break;

2
src/ops/lispops.c
View file

@ -29,9 +29,9 @@
#include "debug.h" #include "debug.h"
#include "dump.h" #include "dump.h"
#include "equal.h" #include "equal.h"
#include "fopen.h"
#include "integer.h" #include "integer.h"
#include "intern.h" #include "intern.h"
#include "io.h"
#include "lispops.h" #include "lispops.h"
#include "print.h" #include "print.h"
#include "read.h" #include "read.h"

1
src/ops/read.c
View file

@ -22,6 +22,7 @@
#include "dump.h" #include "dump.h"
#include "integer.h" #include "integer.h"
#include "intern.h" #include "intern.h"
#include "io.h"
#include "lispops.h" #include "lispops.h"
#include "peano.h" #include "peano.h"
#include "print.h" #include "print.h"