00001 #include "niml_private.h"
00002 
00003 /*************************************************************************/
00004 /********************** Functions for fetching URLs **********************/
00005 /*************************************************************************/
00006 
00007 static int www_debug = 0 ;
00008 #define FAILED     if(www_debug)fprintf(stderr," **FAILED\n")
00009 #define DMESS(s,t) if(www_debug)fprintf(stderr,s,t)
00010 
00011 /*---------------------------------------------------------------------*/
00012 static char tmpdir[512] = "\0" ;
00013 
00014 /*! Set the temporary working file directory from the environment. */
00015 
00016 static void setup_tmpdir(void)
00017 {
00018    char *td ;
00019 
00020    if( tmpdir[0] != '\0' ) return ;
00021 
00022                     td = getenv("TMPDIR")  ;
00023    if( td == NULL ) td = getenv("TEMPDIR") ;
00024 
00025    if( td == NULL || td[0] == '\0' || strlen(td) > 222 ){
00026       strcpy(tmpdir,"/tmp/") ;
00027    } else {
00028       int ltd ;
00029       NI_strncpy(tmpdir,td,511) ;
00030       ltd = strlen(tmpdir) ;
00031       if( tmpdir[ltd-1] != '/' ) strcat(tmpdir,"/") ;
00032    }
00033 }
00034 
00035 /*---------------------------------------------------------------------*/
00036 /*! Open an "http://" URL in host, port, and filename pieces.
00037   Wait up to msec milliseconds for network functions to occur.
00038   If an error occurs, return NULL, otherwise the caller can read
00039   from this NI_stream.
00040 -----------------------------------------------------------------------*/
00041 
00042 static NI_stream_type * open_URL_hpf( char *host, int port,
00043                                       char *file, int msec )
00044 {
00045    NI_stream_type *ns ;
00046    char str[1024] ;
00047    int ii ;
00048 
00049    if( host == NULL || port <= 0 || file == NULL ) return NULL ;
00050 
00051    sprintf(str,"tcp:%s:%d",host,port) ;
00052    DMESS(" ++Opening %s",str);
00053    ns = NI_stream_open( str , "w" ) ;
00054    if( ns == NULL ){ FAILED; return NULL; }
00055    ii = NI_stream_writecheck( ns , msec ) ;
00056    if( ii <= 0 ){ FAILED; NI_stream_close(ns); return NULL; }
00057 
00058    DMESS(" ++GET %s",file);
00059    sprintf(str,"GET %s\n",file) ;                     /* HTTP 0.9 */
00060    ii = NI_stream_write( ns , str , strlen(str) ) ;
00061    if( ii <= 0 ){ FAILED; NI_stream_close(ns); return NULL; }
00062 
00063    ii = NI_stream_readcheck( ns , msec ) ;
00064    if( ii <= 0 ){ FAILED; NI_stream_close(ns); return NULL; }
00065    DMESS("%s"," **OPENED");
00066    return ns ;
00067 }
00068 
00069 /*----------------------------------------------------------------------*/
00070 
00071 #define HTTP     "http://"
00072 #define HTTPLEN  7
00073 
00074 #define FTP      "ftp://"
00075 #define FTPLEN   6
00076 
00077 /*----------------------------------------------------------------------*/
00078 /*! Open an "http://" URL and prepare to read it (but the caller must
00079   actually do the reading).  If NULL is returned, an error occurred.
00080 ------------------------------------------------------------------------*/
00081 
00082 static NI_stream_type * open_URL_http( char *url , int msec )
00083 {
00084   char *s, *h , *file ;
00085   char hostname[1024] ;
00086   int port;
00087   NI_stream_type *ns ;
00088 
00089   /* check inputs */
00090 
00091   if( url == NULL || strstr(url,HTTP) != url ) return NULL ;
00092 
00093   /* parse hostname */
00094 
00095   for( s=url+HTTPLEN , h=hostname ;
00096        (*s != '\0') && (*s != ':') && (*s != '/') ; s++ , h++ ) *h = *s ;
00097 
00098   *h = '\0' ; if( hostname[0] == '\0' ) return NULL ;
00099 
00100   /* parse port number if present */
00101 
00102   port = 0 ;
00103   if( *s == ':' ){ port = strtol( ++s , &h , 10 ) ; s = h ; }
00104   if( port <= 0 ) port = 80 ;
00105 
00106   /* get the file name (keep leading "/") */
00107 
00108   file = (*s == '/') ? s : (char *)"/" ;
00109 
00110   /* do the actual work */
00111 
00112   ns = open_URL_hpf( hostname , port , file , msec ) ;
00113   return ns ;
00114 }
00115 
00116 
00117 /*---------------------------------------------------------------*/
00118 
00119 #define QBUF 4096
00120 
00121 /*---------------------------------------------------------------*/
00122 /*! Read an "http://" URL, with network waits of up to msec
00123   milliseconds allowed.  Returns number of bytes read -- if this
00124   is > 0, then *data will be a pointer to malloc-ed bytes holding
00125   the contents of the file.
00126 
00127   If the file is gzip-ed, then it will be un-gzip-ed before being
00128   loaded into memory.  This uses temporary files in $TMPDIR or
00129   /tmp, which must have space to hold the compressed and
00130   uncompressed file.  If the file is not compressed, then input
00131   is directly to memory and no temporary files are used.
00132 -----------------------------------------------------------------*/
00133 
00134 static int read_URL_http( char *url , int msec , char **data )
00135 {
00136    NI_stream_type *ns ;
00137    char *buf=NULL , *cpt , qbuf[QBUF] , qname[1024] ;
00138    int ii,jj , nall=0 , nuse ;
00139    int cflag , first ;
00140    FILE *cfile=NULL ;
00141 
00142    /* sanity check */
00143 
00144    if( url == NULL || data == NULL || msec < 0 ) return( -1 );
00145 
00146    /* open http channel to get url */
00147 
00148    ns = open_URL_http( url , msec ) ;
00149    if( ns == NULL ){ DMESS("%s","\n"); return( -1 ); }
00150 
00151    /* check if url will be returned gzip-ed */
00152 
00153    ii = strlen(url) ;
00154    if( ii > 3 ){
00155       cpt = url + (ii-3) ; cflag = (strcmp(cpt,".gz") == 0) ;
00156    } else {
00157       cflag = 0 ;
00158    }
00159 
00160    if( cflag ){
00161       setup_tmpdir() ;
00162       strcpy(qname,tmpdir) ; strcat(qname,"ElvisXXXXXX") ;
00163       NI_mktemp(qname) ;
00164       if( qname[0] != '\0' ){
00165          strcat(qname,".gz") ; cfile = fopen( qname , "wb" ) ;
00166          if( cfile == NULL ) cflag == 0 ;
00167       } else {
00168          cflag = 0 ;
00169       }
00170 
00171       if( cflag == 0 ){
00172          DMESS(" **Temp file %s FAILS\n",qname); NI_stream_close(ns); return(-1);
00173       }
00174       DMESS(" ++Temp file=%s",qname);
00175    }
00176 
00177    /* read all of url */
00178 
00179    if( !cflag ){ buf = (char *)malloc( QBUF ) ; nall = QBUF ; }
00180    nuse = 0 ; first = 1 ;
00181 
00182    do{
00183       if(www_debug)fprintf(stderr,".");
00184       ii = NI_stream_readcheck( ns , msec ) ;  /* wait for data to be ready */
00185       if( ii <= 0 ) break ;                    /* quit if no data */
00186       ii = NI_stream_read( ns , qbuf , QBUF ) ;
00187       if( ii <= 0 ) break ;                  /* quit if no data */
00188 
00189       if( first ){                           /* check for "not found" */
00190          if( buf == NULL ){ buf = (char *)malloc(ii) ; }
00191          memcpy( buf , qbuf , ii ) ;
00192          for( jj=0 ; jj < ii ; jj++ ) buf[jj] = toupper(buf[jj]) ;
00193          buf[ii-1] = '\0' ;
00194          cpt = strstr(buf,"NOT FOUND") ;
00195          if( cpt != NULL ){
00196             if( cflag ){ fclose(cfile) ; unlink(qname) ; }
00197             DMESS("%s"," **NOT FOUND\n");
00198             free(buf) ; NI_stream_close(ns) ; return( -1 );
00199          }
00200          first = 0 ;
00201          if( cflag ){ free(buf) ; buf = NULL ; }
00202       }
00203 
00204       if( cflag ){                           /* write to temp file */
00205          nall = fwrite( qbuf , 1 , ii , cfile ) ;
00206          if( nall != ii ){                   /* write failed? */
00207             DMESS("\n** Write to temp file %s FAILED!\n",qname);
00208             fclose(cfile) ; unlink(qname) ;
00209             NI_stream_close(ns) ; return( -1 );
00210          }
00211       } else {                               /* save to buffer */
00212          if( nuse+ii > nall ){               /* enlarge buffer? */
00213             nall += QBUF ;
00214             buf   = (char *)realloc( (void *)buf , nall ) ;
00215          }
00216          memcpy( buf+nuse , qbuf , ii ) ;    /* copy data into buffer */
00217       }
00218       nuse += ii ;                           /* how many bytes so far */
00219    } while(1) ;
00220    NI_stream_close(ns) ;
00221 
00222    /* didn't get anything? */
00223 
00224    if( nuse <= 0 ){
00225       if( cflag ){ fclose(cfile) ; unlink(qname) ; }
00226       else       { free(buf) ; }
00227       FAILED; return(-1);
00228    }
00229    if(www_debug)fprintf(stderr,"!\n");
00230 
00231    /* uncompression time? */
00232 
00233    if( cflag ){
00234       fclose(cfile) ;
00235       sprintf( qbuf , "gzip -dq %s" , qname ) ;     /* execute gzip */
00236       ii = system(qbuf) ;
00237       if( ii != 0 ){ DMESS("%s"," **gzip failed!\n");
00238                      unlink(qname) ; return( -1 );   }  /* gzip failed  */
00239       ii = strlen(qname) ; qname[ii-3] = '\0' ;     /* fix filename */
00240       nuse = NI_filesize( qname ) ;                 /* find how big */
00241       if( nuse <= 0 ){ DMESS("%s"," **gzip failed!\n");
00242                        unlink(qname) ; return( -1 );   }
00243 
00244       cfile = fopen( qname , "rb" ) ;
00245       if( cfile == NULL ){ DMESS("%s"," **gzip failed!\n");
00246                            unlink(qname) ; return( -1 );   }
00247       buf = (char *)malloc(nuse) ;
00248       fread( buf , 1 , nuse , cfile ) ;             /* read file in */
00249       fclose(cfile) ; unlink(qname) ;
00250    }
00251 
00252    /* data is in buf, nuse bytes of it */
00253 
00254    DMESS("%s","\n"); *data = buf ; return( nuse );
00255 }
00256 
00257 /*---------------------------------------------------------------------*/
00258 
00259 static char ftp_name[512] = "anonymous" ;
00260 static char ftp_pwd[512]  = "NIML@nowhere.org" ;
00261 
00262 #if 0
00263 void NI_set_URL_ftp_ident( char *name , char *pwd )
00264 {
00265    int ll ;
00266 
00267    if( name == NULL || pwd == NULL ) return ;
00268 
00269    ll = strlen(name) ; if( ll < 1 || ll > 511 ) return ;
00270    ll = strlen(pwd)  ; if( ll < 1 || ll > 511 ) return ;
00271 
00272    strcpy(ftp_name,name) ; strcpy(ftp_pwd,pwd) ; return ;
00273 }
00274 #endif
00275 
00276 /*---------------------------------------------------------------------*/
00277 /*! Reads an "ftp://" URL, similarly to read_URL_http above;
00278   however, staging is always done through a temporary file.
00279   This function works simply by creating/running a script file to
00280   run the command line ftp program.  Clumsy, but simpler than
00281   implementing the protocol.
00282 -----------------------------------------------------------------------*/
00283 
00284 static int read_URL_ftp( char *url , char **data )
00285 {
00286    char *s, *h , *file , qname[1024] , sname[1024] , *cpt , *buf ;
00287    char hostname[1024] ;
00288    int port , ii , cflag , nuse ;
00289    FILE *sp ;
00290 
00291    /* sanity check */
00292 
00293    if( url == NULL || data == NULL || strstr(url,FTP) != url ) return( -1 );
00294 
00295    /* parse hostname */
00296 
00297    for( s=url+FTPLEN , h=hostname ;
00298         (*s != '\0') && (*s != ':') && (*s != '/') ; s++ , h++ ) *h = *s ;
00299 
00300    *h = '\0' ; if( hostname[0] == '\0' ) return( -1 );
00301 
00302    /* parse port number, if present */
00303 
00304    port = 0 ;
00305    if( *s == ':' ){ port = strtol( ++s , &h , 10 ) ; s = h ; }
00306 
00307    /* get the file name (strip off leading "/") */
00308 
00309    if( *s == '/' ){
00310       file = s+1 ; if( file[0] == '\0' ) return( -1 );
00311    } else {
00312                                          return( -1 );
00313    }
00314 
00315    /* check if file will be returned gzip-ed */
00316 
00317    ii = strlen(file) ;
00318    if( ii > 3 ){
00319       cpt = file + (ii-3) ; cflag = (strcmp(cpt,".gz") == 0) ;
00320    } else {
00321       cflag = 0 ;
00322    }
00323 
00324    /* make name for output file */
00325 
00326    setup_tmpdir() ;
00327    strcpy(qname,tmpdir) ; strcat(qname,"EthelXXXXXX") ;
00328    NI_mktemp(qname) ;
00329    if( qname[0] == '\0' ) return( -1 );
00330    if( cflag ) strcat(qname,".gz") ;
00331 
00332    /* write the script file that will be used to run ftp */
00333 
00334    strcpy(sname,tmpdir) ; strcat(sname,"DahmerXXXXXX") ;
00335    NI_mktemp(sname) ;          if( sname[0] == '\0' ) return( -1 );
00336    sp = fopen( sname , "w" ) ; if( sp == NULL )       return( -1 );
00337 
00338    fprintf( sp , "#!/bin/sh\n" ) ;
00339    fprintf( sp , "ftp -n << EEEEE &> /dev/null\n") ;
00340    if( port > 0 )
00341       fprintf( sp , "open %s %d\n" , hostname , port ) ;
00342    else
00343       fprintf( sp , "open %s\n" , hostname ) ;
00344    fprintf( sp , "user %s %s\n" , ftp_name, ftp_pwd ) ;
00345    fprintf( sp , "binary\n" ) ;
00346    fprintf( sp , "get %s %s\n" , file , qname ) ;
00347    fprintf( sp , "bye\n" ) ;
00348    fprintf( sp , "EEEEE\n" ) ;
00349    fprintf( sp , "exit\n" ) ;
00350    fclose( sp ) ;
00351    chmod( sname , S_IRUSR | S_IWUSR | S_IXUSR ) ;
00352 
00353    /* execute the script, then delete it */
00354 
00355    system( sname ) ; unlink( sname ) ;
00356 
00357    /* check the size of the output file */
00358 
00359    nuse = NI_filesize( qname ) ;
00360    if( nuse <= 0 ){ unlink(qname) ; return( -1 ); }
00361 
00362    /* uncompress the file, if needed */
00363 
00364    if( cflag ){
00365       sprintf( sname , "gzip -dq %s" , qname ) ;    /* execute gzip */
00366       ii = system(sname) ;
00367       if( ii != 0 ){ unlink(qname) ; return( -1 ); }  /* gzip failed  */
00368       ii = strlen(qname) ; qname[ii-3] = '\0' ;     /* fix filename */
00369       nuse = NI_filesize( qname ) ;                 /* find how big */
00370       if( nuse <= 0 ){ unlink(qname) ; return( -1 ); }
00371    }
00372 
00373    /* suck the file into memory */
00374 
00375    sp = fopen( qname , "rb" ) ;
00376    if( sp == NULL ){ unlink(qname) ; return( -1 ); }
00377    buf = (char *)malloc(nuse); if( buf == NULL ){ unlink(qname); return( -1 ); }
00378 
00379    fread( buf , 1 , nuse , sp ) ;  /* AT LAST! */
00380    fclose(sp) ; unlink(qname) ;
00381 
00382    /* data is in buf, nuse bytes of it */
00383 
00384    *data = buf ; return( nuse );
00385 }
00386 
00387 /*-------------------------------------------------------------------*/
00388 /*! Read a URL (ftp:// or http://) into memory.  The return value
00389    is the number of bytes read, and *data points to the data.
00390    If the return value is negative, then something bad happened.
00391 
00392    Example:
00393      -  int nn ;
00394      -  char *data ;
00395      -  nn = NI_read_URL( "http://zork.ork/oog" , &data ) ;
00396      -  if( nn <= 0 ){ **failure** }
00397      -  else         { **data[0..nn-1] can be used for something** }
00398 ---------------------------------------------------------------------*/
00399 
00400 int NI_read_URL( char *url , char **data )
00401 {
00402    int nn ;
00403    if( url == NULL || data == NULL ) return( -1 );
00404 
00405    if( getenv("NIML_WWW_DEBUG") != NULL ) www_debug = 1 ;
00406 
00407    if( strstr(url,HTTP) == url ){
00408       nn = read_URL_http( url , 4444 , data ) ; return(nn) ;
00409    }
00410 
00411    else if( strstr(url,FTP) == url ){
00412       nn = read_URL_ftp( url , data ) ; return(nn) ;
00413    }
00414 
00415    return( -1 );
00416 }
00417 
00418 /*------------------------------------------------------------------*/
00419 /*! Read a URL and save it to disk in tmpdir.  The filename
00420   it is saved in is returned in the malloc-ed space *tname.
00421   The byte count is the return value of the function;
00422   if <= 0, then an error transpired (and *tname is not set).
00423 
00424   Example:
00425     -  int nn ;
00426     -  char *tname ;
00427     -  nn = NI_read_URL_tmpdir( "ftp://whoople.oog/zorkon" , &tname ) ;
00428     -  if( nn <= 0 ){ **failure** }
00429     -  else         { **you can read file tname at your leisure** }
00430 --------------------------------------------------------------------*/
00431 
00432 int NI_read_URL_tmpdir( char *url , char **tname )
00433 {
00434    int nn , ll ;
00435    char *data , *fname , *tt ;
00436    FILE *fp ;
00437 
00438    if( url == NULL || tname == NULL ) return( -1 );
00439 
00440    nn = NI_read_URL( url , &data ) ;  /* get the data into memory */
00441    if( nn <= 0 ) return( -1 );        /* bad */
00442 
00443    /* make the output filename */
00444 
00445    setup_tmpdir() ;
00446    fname = (char *)malloc(strlen(url)+strlen(tmpdir)+1) ;
00447    tt    = trailname(url,0) ;
00448    strcpy(fname,tmpdir) ; strcat(fname,tt) ; ll = strlen(fname) ;
00449    if( ll > 3 && strcmp(fname+(ll-3),".gz") == 0 ) fname[ll-3] = '\0' ;
00450 
00451    /* open and write output */
00452 
00453    fp = fopen( fname , "wb" ) ;
00454    if( fp == NULL ){
00455      fprintf(stderr,"** Can't open temporary file %s\n",fname);
00456      free(data) ; return( -1 );
00457    }
00458    ll = fwrite(data,1,nn,fp) ; fclose(fp) ; free(data) ;
00459    if( ll != nn ){ unlink(fname); return( -1 ); } /* write failed */
00460 
00461    *tname = fname ; return( nn );
00462 }