simon/post-scarcity
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

Setting up medatata works...

Browse source 
And the `inspect` function correctly shows it. However, the `metadata` function segfaults.
This commit is contained in:
Simon Brooke 2019-01-29 18:31:30 +00:00
parent 10098a83bf
commit eb394d153f
16 changed files with 866 additions and 580 deletions
Download patch file Download diff file Expand all files Collapse all files

774
src/io/fopen.c
View file

@ -47,392 +47,369 @@
#include <curl/curl.h>
enum fcurl_type_e {
CFTYPE_NONE = 0,
CFTYPE_FILE = 1,
CFTYPE_CURL = 2
};
#include "fopen.h"
#ifdef FOPEN_STANDALONE
CURLSH *io_share;
#else
#include "io.h"
#include "consspaceobject.h"
#endif
struct fcurl_data
{
enum fcurl_type_e type; /* type of handle */
union {
CURL *curl;
FILE *file;
} handle; /* handle */
char *buffer; /* buffer to store cached data*/
size_t buffer_len; /* currently allocated buffers length */
size_t buffer_pos; /* end of data in buffer*/
int still_running; /* Is background url fetch still in progress */
};
typedef struct fcurl_data URL_FILE;
/* exported functions */
URL_FILE *url_fopen(const char *url, const char *operation);
int url_fclose(URL_FILE *file);
int url_feof(URL_FILE *file);
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
char *url_fgets(char *ptr, size_t size, URL_FILE *file);
void url_rewind(URL_FILE *file);
URL_FILE *url_fopen( const char *url, const char *operation );
int url_fclose( URL_FILE * file );
int url_feof( URL_FILE * file );
size_t url_fread( void *ptr, size_t size, size_t nmemb, URL_FILE * file );
char *url_fgets( char *ptr, size_t size, URL_FILE * file );
void url_rewind( URL_FILE * file );
/* we use a global one for convenience */
static CURLM *multi_handle;
/* curl calls this routine to get more data */
static size_t write_callback(char *buffer,
size_t size,
size_t nitems,
void *userp)
{
char *newbuff;
size_t rembuff;
static size_t write_callback( char *buffer,
size_t size, size_t nitems, void *userp ) {
char *newbuff;
size_t rembuff;
URL_FILE *url = (URL_FILE *)userp;
size *= nitems;
URL_FILE *url = ( URL_FILE * ) userp;
size *= nitems;
rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */
rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */
if(size > rembuff) {
/* not enough space in buffer */
newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff));
if(newbuff == NULL) {
fprintf(stderr, "callback buffer grow failed\n");
size = rembuff;
if ( size > rembuff ) {
/* not enough space in buffer */
newbuff = realloc( url->buffer, url->buffer_len + ( size - rembuff ) );
if ( newbuff == NULL ) {
fprintf( stderr, "callback buffer grow failed\n" );
size = rembuff;
} else {
/* realloc succeeded increase buffer size */
url->buffer_len += size - rembuff;
url->buffer = newbuff;
}
}
else {
/* realloc succeeded increase buffer size*/
url->buffer_len += size - rembuff;
url->buffer = newbuff;
}
}
memcpy(&url->buffer[url->buffer_pos], buffer, size);
url->buffer_pos += size;
memcpy( &url->buffer[url->buffer_pos], buffer, size );
url->buffer_pos += size;
return size;
return size;
}
/* use to attempt to fill the read buffer up to requested number of bytes */
static int fill_buffer(URL_FILE *file, size_t want)
{
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
struct timeval timeout;
int rc;
CURLMcode mc; /* curl_multi_fdset() return code */
static int fill_buffer( URL_FILE * file, size_t want ) {
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
struct timeval timeout;
int rc;
CURLMcode mc; /* curl_multi_fdset() return code */
/* only attempt to fill buffer if transactions still running and buffer
* doesn't exceed required size already
*/
if((!file->still_running) || (file->buffer_pos > want))
return 0;
/* only attempt to fill buffer if transactions still running and buffer
* doesn't exceed required size already
*/
if ( ( !file->still_running ) || ( file->buffer_pos > want ) )
return 0;
/* attempt to fill buffer */
do {
int maxfd = -1;
long curl_timeo = -1;
/* attempt to fill buffer */
do {
int maxfd = -1;
long curl_timeo = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
FD_ZERO( &fdread );
FD_ZERO( &fdwrite );
FD_ZERO( &fdexcep );
/* set a suitable timeout to fail on */
timeout.tv_sec = 60; /* 1 minute */
timeout.tv_usec = 0;
/* set a suitable timeout to fail on */
timeout.tv_sec = 60; /* 1 minute */
timeout.tv_usec = 0;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo >= 0) {
timeout.tv_sec = curl_timeo / 1000;
if(timeout.tv_sec > 1)
timeout.tv_sec = 1;
else
timeout.tv_usec = (curl_timeo % 1000) * 1000;
}
curl_multi_timeout( multi_handle, &curl_timeo );
if ( curl_timeo >= 0 ) {
timeout.tv_sec = curl_timeo / 1000;
if ( timeout.tv_sec > 1 )
timeout.tv_sec = 1;
else
timeout.tv_usec = ( curl_timeo % 1000 ) * 1000;
}
/* get file descriptors from the transfers */
mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
/* get file descriptors from the transfers */
mc = curl_multi_fdset( multi_handle, &fdread, &fdwrite, &fdexcep,
&maxfd );
if(mc != CURLM_OK) {
fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
break;
}
if ( mc != CURLM_OK ) {
fprintf( stderr, "curl_multi_fdset() failed, code %d.\n", mc );
break;
}
/* On success the value of maxfd is guaranteed to be >= -1. We call
select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
to sleep 100ms, which is the minimum suggested value in the
curl_multi_fdset() doc. */
/* On success the value of maxfd is guaranteed to be >= -1. We call
select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
to sleep 100ms, which is the minimum suggested value in the
curl_multi_fdset() doc. */
if(maxfd == -1) {
if ( maxfd == -1 ) {
#ifdef _WIN32
Sleep(100);
rc = 0;
Sleep( 100 );
rc = 0;
#else
/* Portable sleep for platforms other than Windows. */
struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
rc = select(0, NULL, NULL, NULL, &wait);
/* Portable sleep for platforms other than Windows. */
struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
rc = select( 0, NULL, NULL, NULL, &wait );
#endif
}
else {
/* Note that on some platforms 'timeout' may be modified by select().
If you need access to the original value save a copy beforehand. */
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
}
} else {
/* Note that on some platforms 'timeout' may be modified by select().
If you need access to the original value save a copy beforehand. */
rc = select( maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout );
}
switch(rc) {
case -1:
/* select error */
break;
switch ( rc ) {
case -1:
/* select error */
break;
case 0:
default:
/* timeout or readable/writable sockets */
curl_multi_perform(multi_handle, &file->still_running);
break;
}
} while(file->still_running && (file->buffer_pos < want));
return 1;
case 0:
default:
/* timeout or readable/writable sockets */
curl_multi_perform( multi_handle, &file->still_running );
break;
}
} while ( file->still_running && ( file->buffer_pos < want ) );
return 1;
}
/* use to remove want bytes from the front of a files buffer */
static int use_buffer(URL_FILE *file, size_t want)
{
/* sort out buffer */
if((file->buffer_pos - want) <= 0) {
/* ditch buffer - write will recreate */
free(file->buffer);
file->buffer = NULL;
file->buffer_pos = 0;
file->buffer_len = 0;
}
else {
/* move rest down make it available for later */
memmove(file->buffer,
&file->buffer[want],
(file->buffer_pos - want));
static int use_buffer( URL_FILE * file, size_t want ) {
/* sort out buffer */
if ( ( file->buffer_pos - want ) <= 0 ) {
/* ditch buffer - write will recreate */
free( file->buffer );
file->buffer = NULL;
file->buffer_pos = 0;
file->buffer_len = 0;
} else {
/* move rest down make it available for later */
memmove( file->buffer,
&file->buffer[want], ( file->buffer_pos - want ) );
file->buffer_pos -= want;
}
return 0;
}
URL_FILE *url_fopen(const char *url, const char *operation)
{
/* this code could check for URLs or types in the 'url' and
basically use the real fopen() for standard files */
URL_FILE *file;
(void)operation;
file = calloc(1, sizeof(URL_FILE));
if(!file)
return NULL;
file->handle.file = fopen(url, operation);
if(file->handle.file)
file->type = CFTYPE_FILE; /* marked as URL */
else {
file->type = CFTYPE_CURL; /* marked as URL */
file->handle.curl = curl_easy_init();
curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
if(!multi_handle)
multi_handle = curl_multi_init();
curl_multi_add_handle(multi_handle, file->handle.curl);
/* lets start the fetch */
curl_multi_perform(multi_handle, &file->still_running);
if((file->buffer_pos == 0) && (!file->still_running)) {
/* if still_running is 0 now, we should return NULL */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* cleanup */
curl_easy_cleanup(file->handle.curl);
free(file);
file = NULL;
file->buffer_pos -= want;
}
}
return file;
return 0;
}
int url_fclose(URL_FILE *file)
{
int ret = 0;/* default is good return */
URL_FILE *url_fopen( const char *url, const char *operation ) {
/* this code could check for URLs or types in the 'url' and
basically use the real fopen() for standard files */
switch(file->type) {
case CFTYPE_FILE:
ret = fclose(file->handle.file); /* passthrough */
break;
URL_FILE *file;
( void ) operation;
case CFTYPE_CURL:
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
file = calloc( 1, sizeof( URL_FILE ) );
if ( !file )
return NULL;
/* cleanup */
curl_easy_cleanup(file->handle.curl);
break;
file->handle.file = fopen( url, operation );
if ( file->handle.file )
file->type = CFTYPE_FILE; /* marked as URL */
default: /* unknown or supported type - oh dear */
ret = EOF;
errno = EBADF;
break;
}
else {
file->type = CFTYPE_CURL; /* marked as URL */
file->handle.curl = curl_easy_init( );
free(file->buffer);/* free any allocated buffer space */
free(file);
curl_easy_setopt( file->handle.curl, CURLOPT_URL, url );
curl_easy_setopt( file->handle.curl, CURLOPT_WRITEDATA, file );
curl_easy_setopt( file->handle.curl, CURLOPT_VERBOSE, 0L );
curl_easy_setopt( file->handle.curl, CURLOPT_WRITEFUNCTION,
write_callback );
/* use the share object */
curl_easy_setopt(file->handle.curl, CURLOPT_SHARE, io_share);
return ret;
if ( !multi_handle )
multi_handle = curl_multi_init( );
curl_multi_add_handle( multi_handle, file->handle.curl );
/* lets start the fetch */
curl_multi_perform( multi_handle, &file->still_running );
if ( ( file->buffer_pos == 0 ) && ( !file->still_running ) ) {
/* if still_running is 0 now, we should return NULL */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle( multi_handle, file->handle.curl );
/* cleanup */
curl_easy_cleanup( file->handle.curl );
free( file );
file = NULL;
}
}
return file;
}
int url_feof(URL_FILE *file)
{
int ret = 0;
int url_fclose( URL_FILE * file ) {
int ret = 0; /* default is good return */
switch(file->type) {
case CFTYPE_FILE:
ret = feof(file->handle.file);
break;
switch ( file->type ) {
case CFTYPE_FILE:
ret = fclose( file->handle.file ); /* passthrough */
break;
case CFTYPE_CURL:
if((file->buffer_pos == 0) && (!file->still_running))
ret = 1;
break;
case CFTYPE_CURL:
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle( multi_handle, file->handle.curl );
default: /* unknown or supported type - oh dear */
ret = -1;
errno = EBADF;
break;
}
return ret;
}
/* cleanup */
curl_easy_cleanup( file->handle.curl );
break;
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
{
size_t want;
switch(file->type) {
case CFTYPE_FILE:
want = fread(ptr, size, nmemb, file->handle.file);
break;
case CFTYPE_CURL:
want = nmemb * size;
fill_buffer(file, want);
/* check if there's data in the buffer - if not fill_buffer()
* either errored or EOF */
if(!file->buffer_pos)
return 0;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
use_buffer(file, want);
want = want / size; /* number of items */
break;
default: /* unknown or supported type - oh dear */
want = 0;
errno = EBADF;
break;
}
return want;
}
char *url_fgets(char *ptr, size_t size, URL_FILE *file)
{
size_t want = size - 1;/* always need to leave room for zero termination */
size_t loop;
switch(file->type) {
case CFTYPE_FILE:
ptr = fgets(ptr, (int)size, file->handle.file);
break;
case CFTYPE_CURL:
fill_buffer(file, want);
/* check if there's data in the buffer - if not fill either errored or
* EOF */
if(!file->buffer_pos)
return NULL;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/*buffer contains data */
/* look for newline or eof */
for(loop = 0; loop < want; loop++) {
if(file->buffer[loop] == '\n') {
want = loop + 1;/* include newline */
break;
}
default: /* unknown or supported type - oh dear */
ret = EOF;
errno = EBADF;
break;
}
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
ptr[want] = 0;/* always null terminate */
free( file->buffer ); /* free any allocated buffer space */
free( file );
use_buffer(file, want);
break;
default: /* unknown or supported type - oh dear */
ptr = NULL;
errno = EBADF;
break;
}
return ptr;/*success */
return ret;
}
void url_rewind(URL_FILE *file)
{
switch(file->type) {
case CFTYPE_FILE:
rewind(file->handle.file); /* passthrough */
break;
int url_feof( URL_FILE * file ) {
int ret = 0;
case CFTYPE_CURL:
/* halt transaction */
curl_multi_remove_handle(multi_handle, file->handle.curl);
switch ( file->type ) {
case CFTYPE_FILE:
ret = feof( file->handle.file );
break;
/* restart */
curl_multi_add_handle(multi_handle, file->handle.curl);
case CFTYPE_CURL:
if ( ( file->buffer_pos == 0 ) && ( !file->still_running ) )
ret = 1;
break;
/* ditch buffer - write will recreate - resets stream pos*/
free(file->buffer);
file->buffer = NULL;
file->buffer_pos = 0;
file->buffer_len = 0;
default: /* unknown or supported type - oh dear */
ret = -1;
errno = EBADF;
break;
}
return ret;
}
break;
size_t url_fread( void *ptr, size_t size, size_t nmemb, URL_FILE * file ) {
size_t want;
default: /* unknown or supported type - oh dear */
break;
}
switch ( file->type ) {
case CFTYPE_FILE:
want = fread( ptr, size, nmemb, file->handle.file );
break;
case CFTYPE_CURL:
want = nmemb * size;
fill_buffer( file, want );
/* check if there's data in the buffer - if not fill_buffer()
* either errored or EOF */
if ( !file->buffer_pos )
return 0;
/* ensure only available data is considered */
if ( file->buffer_pos < want )
want = file->buffer_pos;
/* xfer data to caller */
memcpy( ptr, file->buffer, want );
use_buffer( file, want );
want = want / size; /* number of items */
break;
default: /* unknown or supported type - oh dear */
want = 0;
errno = EBADF;
break;
}
return want;
}
char *url_fgets( char *ptr, size_t size, URL_FILE * file ) {
size_t want = size - 1; /* always need to leave room for zero termination */
size_t loop;
switch ( file->type ) {
case CFTYPE_FILE:
ptr = fgets( ptr, ( int ) size, file->handle.file );
break;
case CFTYPE_CURL:
fill_buffer( file, want );
/* check if there's data in the buffer - if not fill either errored or
* EOF */
if ( !file->buffer_pos )
return NULL;
/* ensure only available data is considered */
if ( file->buffer_pos < want )
want = file->buffer_pos;
/*buffer contains data */
/* look for newline or eof */
for ( loop = 0; loop < want; loop++ ) {
if ( file->buffer[loop] == '\n' ) {
want = loop + 1; /* include newline */
break;
}
}
/* xfer data to caller */
memcpy( ptr, file->buffer, want );
ptr[want] = 0; /* always null terminate */
use_buffer( file, want );
break;
default: /* unknown or supported type - oh dear */
ptr = NULL;
errno = EBADF;
break;
}
return ptr; /*success */
}
void url_rewind( URL_FILE * file ) {
switch ( file->type ) {
case CFTYPE_FILE:
rewind( file->handle.file ); /* passthrough */
break;
case CFTYPE_CURL:
/* halt transaction */
curl_multi_remove_handle( multi_handle, file->handle.curl );
/* restart */
curl_multi_add_handle( multi_handle, file->handle.curl );
/* ditch buffer - write will recreate - resets stream pos */
free( file->buffer );
file->buffer = NULL;
file->buffer_pos = 0;
file->buffer_len = 0;
break;
default: /* unknown or supported type - oh dear */
break;
}
}
#ifdef FOPEN_STANDALONE
@ -443,104 +420,103 @@ void url_rewind(URL_FILE *file)
/* Small main program to retrieve from a url using fgets and fread saving the
* output to two test files (note the fgets method will corrupt binary files if
* they contain 0 chars */
int main(int argc, char *argv[])
{
URL_FILE *handle;
FILE *outf;
int main( int argc, char *argv[] ) {
URL_FILE *handle;
FILE *outf;
size_t nread;
char buffer[256];
const char *url;
size_t nread;
char buffer[256];
const char *url;
CURL *curl;
CURLcode res;
CURL *curl;
CURLcode res;
curl_global_init(CURL_GLOBAL_DEFAULT);
curl_global_init( CURL_GLOBAL_DEFAULT );
curl = curl_easy_init();
curl = curl_easy_init( );
if(argc < 2)
url = "http://192.168.7.3/testfile";/* default to testurl */
else
url = argv[1];/* use passed url */
if ( argc < 2 )
url = "http://192.168.7.3/testfile"; /* default to testurl */
else
url = argv[1]; /* use passed url */
/* copy from url line by line with fgets */
outf = fopen(FGETSFILE, "wb+");
if(!outf) {
perror("couldn't open fgets output file\n");
return 1;
}
/* copy from url line by line with fgets */
outf = fopen( FGETSFILE, "wb+" );
if ( !outf ) {
perror( "couldn't open fgets output file\n" );
return 1;
}
handle = url_fopen(url, "r");
if(!handle) {
printf("couldn't url_fopen() %s\n", url);
fclose(outf);
return 2;
}
handle = url_fopen( url, "r" );
if ( !handle ) {
printf( "couldn't url_fopen() %s\n", url );
fclose( outf );
return 2;
}
while(!url_feof(handle)) {
url_fgets(buffer, sizeof(buffer), handle);
fwrite(buffer, 1, strlen(buffer), outf);
}
while ( !url_feof( handle ) ) {
url_fgets( buffer, sizeof( buffer ), handle );
fwrite( buffer, 1, strlen( buffer ), outf );
}
url_fclose(handle);
url_fclose( handle );
fclose(outf);
fclose( outf );
/* Copy from url with fread */
outf = fopen(FREADFILE, "wb+");
if(!outf) {
perror("couldn't open fread output file\n");
return 1;
}
/* Copy from url with fread */
outf = fopen( FREADFILE, "wb+" );
if ( !outf ) {
perror( "couldn't open fread output file\n" );
return 1;
}
handle = url_fopen("testfile", "r");
if(!handle) {
printf("couldn't url_fopen() testfile\n");
fclose(outf);
return 2;
}
handle = url_fopen( "testfile", "r" );
if ( !handle ) {
printf( "couldn't url_fopen() testfile\n" );
fclose( outf );
return 2;
}
do {
nread = url_fread(buffer, 1, sizeof(buffer), handle);
fwrite(buffer, 1, nread, outf);
} while(nread);
do {
nread = url_fread( buffer, 1, sizeof( buffer ), handle );
fwrite( buffer, 1, nread, outf );
} while ( nread );
url_fclose(handle);
url_fclose( handle );
fclose(outf);
fclose( outf );
/* Test rewind */
outf = fopen(REWINDFILE, "wb+");
if(!outf) {
perror("couldn't open fread output file\n");
return 1;
}
/* Test rewind */
outf = fopen( REWINDFILE, "wb+" );
if ( !outf ) {
perror( "couldn't open fread output file\n" );
return 1;
}
handle = url_fopen("testfile", "r");
if(!handle) {
printf("couldn't url_fopen() testfile\n");
fclose(outf);
return 2;
}
handle = url_fopen( "testfile", "r" );
if ( !handle ) {
printf( "couldn't url_fopen() testfile\n" );
fclose( outf );
return 2;
}
nread = url_fread(buffer, 1, sizeof(buffer), handle);
fwrite(buffer, 1, nread, outf);
url_rewind(handle);
nread = url_fread( buffer, 1, sizeof( buffer ), handle );
fwrite( buffer, 1, nread, outf );
url_rewind( handle );
buffer[0]='\n';
fwrite(buffer, 1, 1, outf);
buffer[0] = '\n';
fwrite( buffer, 1, 1, outf );
nread = url_fread(buffer, 1, sizeof(buffer), handle);
fwrite(buffer, 1, nread, outf);
nread = url_fread( buffer, 1, sizeof( buffer ), handle );
fwrite( buffer, 1, nread, outf );
url_fclose(handle);
url_fclose( handle );
fclose(outf);
fclose( outf );
return 0;/* all done */
return 0; /* all done */
}
#endif

192
src/io/io.c
View file

@ -8,6 +8,17 @@
*/
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
/*
* wide characters
*/
#include <wchar.h>
#include <wctype.h>
#include <curl/curl.h>
#include "conspage.h"
#include "consspaceobject.h"
@ -15,12 +26,42 @@
#include "fopen.h"
#include "lispops.h"
/**
* The sharing hub for all connections. TODO: Ultimately this probably doesn't
* work for a multi-user environment and we will need one sharing hub for each
* user, or else we will need to not share at least cookies and ssl sessions.
*/
CURLSH *io_share;
/**
* Allow a one-character unget facility. This may not be enough - we may need
* to allocate a buffer.
*/
wint_t ungotten = 0;
/**
* Initialise the I/O subsystem.
*
* @return 0 on success; any other value means failure.
*/
int io_init() {
CURL *curl;
CURLcode res;
int result = curl_global_init( CURL_GLOBAL_SSL );
io_share = curl_share_init();
if (result == 0) {
curl_share_setopt(io_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
curl_share_setopt(io_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE );
curl_share_setopt(io_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS );
curl_share_setopt(io_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION );
curl_share_setopt(io_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_PSL );
}
return result;
}
/**
* Convert this lisp string-like-thing (also works for symbols, and, later
* keywords) into a UTF-8 string. NOTE that the returned value has been
@ -107,13 +148,15 @@ wint_t url_fgetwc( URL_FILE * input ) {
size_t count = 0;
debug_print( L"url_fgetwc: about to call url_fgets\n", DEBUG_IO );
debug_print( L"url_fgetwc: about to call url_fgets\n",
DEBUG_IO );
url_fgets( cbuff, 2, input );
debug_print( L"url_fgetwc: back from url_fgets\n", DEBUG_IO );
debug_print( L"url_fgetwc: back from url_fgets\n",
DEBUG_IO );
int c = ( int ) cbuff[0];
debug_printf( DEBUG_IO,
L"url_fgetwc: cbuff is '%s'; (first) character = %d (%c)\n",
cbuff, c, c & 0xf7 );
L"url_fgetwc: cbuff is '%s'; (first) character = %d (%c)\n",
cbuff, c, c & 0xf7 );
/* The value of each individual byte indicates its UTF-8 function, as follows:
*
* 00 to 7F hex (0 to 127): first and only byte of a sequence.
@ -133,7 +176,7 @@ wint_t url_fgetwc( URL_FILE * input ) {
}
if ( count > 1 ) {
url_fgets( (char *)&cbuff[1], count, input );
url_fgets( ( char * ) &cbuff[1], count, input );
}
mbstowcs( wbuff, cbuff, 1 ); //(char *)(&input->buffer[input->buffer_pos]), 1 );
result = wbuff[0];
@ -163,18 +206,6 @@ wint_t url_ungetwc( wint_t wc, URL_FILE * input ) {
case CFTYPE_CURL:{
ungotten = wc;
// wchar_t *wbuff = calloc( 2, sizeof( wchar_t ) );
// char *cbuff = calloc( 5, sizeof( char ) );
//
// wbuff[0] = wc;
// result = wcstombs( cbuff, wbuff, 1 );
//
// input->buffer_pos -= strlen( cbuff );
//
// free( cbuff );
// free( wbuff );
//
// result = result > 0 ? wc : result;
break;
case CFTYPE_NONE:
break;
@ -212,6 +243,85 @@ lisp_close( struct stack_frame *frame, struct cons_pointer frame_pointer,
return result;
}
int index_of( char c, char * s) {
int i;
for (i = 0; s[i] != c && s[i] != 0; i++);
return s[i] == c ? i : -1;
}
char * trim(char *s) {
int i;
for (i = strlen(s); (isblank(s[i]) || iscntrl(s[i])) && i > -1; i--) {
s[i] = (char) 0;
}
for (i = 0; isblank(s[i]) && s[i] != 0; i++);
return (char *)&s[i];
}
/**
* Callback to assemble metadata for a URL stream. This is naughty because
* it modifies data, but it's really the only way to create metadata.
*/
static size_t write_meta_callback(void *ptr, size_t size, size_t nmemb, struct cons_pointer stream)
{
struct cons_space_object * cell = &pointer2cell(stream);
if (strncmp(&cell->tag.bytes[0], READTAG, 4) ||
strncmp(&cell->tag.bytes[0], WRITETAG, 4)) {
char * s = (char *)ptr;
int offset = index_of (':', ptr);
if (offset != -1) {
s[offset] = (char)0;
char * name = s;
char * value = trim( &s[++offset]);
wchar_t * wname = calloc(strlen(name), sizeof(wchar_t));
wchar_t * wvalue = calloc(strlen(value), sizeof(wchar_t));
mbstowcs(wname, name, strlen(name));
mbstowcs(wvalue, value, strlen(value));
cell->payload.stream.meta = make_cons(
make_cons(
c_string_to_lisp_keyword( wname),
c_string_to_lisp_string(wvalue)),
cell->payload.stream.meta);
debug_printf( DEBUG_IO, L"write_meta_callback: added header '%s': value '%s'\n", name, value);
}
} else {
debug_print( L"Pointer passed to write_meta_callback did not point to a stream: ", DEBUG_IO);
debug_dump_object(stream, DEBUG_IO);
}
return nmemb;
}
void collect_meta( struct cons_pointer stream, struct cons_pointer url ) {
URL_FILE * s = pointer2cell(stream).payload.stream.stream;
switch ( s->type ) {
case CFTYPE_NONE:
break;
case CFTYPE_FILE:
/* don't know whether you can get metadata on an open stream in C,
* although we could of course get it from the URL */
break;
case CFTYPE_CURL:
curl_easy_setopt( s->handle.curl, CURLOPT_VERBOSE, 1L );
curl_easy_setopt( s->handle.curl, CURLOPT_HEADER, 1L );
curl_easy_setopt( s->handle.curl, CURLOPT_HEADERFUNCTION, write_meta_callback);
curl_easy_setopt( s->handle.curl, CURLOPT_HEADERDATA, stream);
break;
}
}
/**
* Function: return a stream open on the URL indicated by the first argument;
* if a second argument is present and is non-nil, open it for reading. At
@ -228,28 +338,38 @@ lisp_close( struct stack_frame *frame, struct cons_pointer frame_pointer,
* on my stream, if any, else NIL.
*/
struct cons_pointer
lisp_open( struct stack_frame *frame, struct cons_pointer frame_pointer,
struct cons_pointer env ) {
struct cons_pointer result = NIL;
lisp_open( struct stack_frame *frame, struct cons_pointer frame_pointer,
struct cons_pointer env ) {
struct cons_pointer result = NIL;
if ( stringp( frame->arg[0] ) ) {
char *url = lisp_string_to_c_string( frame->arg[0] );
if ( stringp( frame->arg[0] ) ) {
struct cons_pointer meta =
make_cons( make_cons(
c_string_to_lisp_keyword( L"url" ),
frame->arg[0] ),
NIL );
if ( nilp( frame->arg[1] ) ) {
result = make_read_stream( url_fopen( url, "r" ) );
} else {
// TODO: anything more complex is a problem for another day.
result = make_write_stream( url_fopen( url, "w" ) );
}
char *url = lisp_string_to_c_string( frame->arg[0] );
free( url );
if ( pointer2cell( result ).payload.stream.stream == NULL ) {
result = NIL;
}
if ( nilp( frame->arg[1] ) ) {
URL_FILE *stream = url_fopen( url, "r" );
result = make_read_stream( stream, meta );
} else {
// TODO: anything more complex is a problem for another day.
URL_FILE *stream = url_fopen( url, "w" );
result = make_write_stream( stream, meta);
}
return result;
free( url );
if ( pointer2cell( result ).payload.stream.stream == NULL ) {
result = NIL;
} else {
collect_meta( result, frame->arg[0]);
}
}
return result;
}
/**
@ -272,8 +392,8 @@ lisp_read_char( struct stack_frame *frame, struct cons_pointer frame_pointer,
if ( readp( frame->arg[0] ) ) {
result =
make_string( url_fgetwc
( pointer2cell( frame->arg[0] ).payload.stream.
stream ), NIL );
( pointer2cell( frame->arg[0] ).payload.
stream.stream ), NIL );
}
return result;

6
src/io/io.h
View file

@ -10,6 +10,12 @@
#ifndef __psse_io_h
#define __psse_io_h
#include <curl/curl.h>
#include "consspaceobject.h"
extern CURLSH *io_share;
int io_init();
URL_FILE *file_to_url_file( FILE * f );
wint_t url_fgetwc( URL_FILE * input );