/Users/barrand/private/dev/softinex/old/inexlib-1.2/inlib/inlib/net/http

Go to the documentation of this file.

// Copyright (C) 2010, Guy Barrand. All rights reserved.
// See the file inlib.license for terms.

#ifndef inlib_http
#define inlib_http

// WARNING : still experimental.

//inheritance :
#include "inet_socket"

#include "../typedefs"
#include "../file"
#include "../cstr"

namespace inlib {
namespace net {

class http : public inet_socket {
public:
  static bool parse(const std::string& a_url,std::string& a_host,std::string& a_path) {
    if(a_url.substr(0,7)!="http://") return false;
    std::string s = a_url.substr(7,a_url.size()-7);
    std::string::size_type pos = s.find('/');
    if(pos==std::string::npos) return false;
    a_host = s.substr(0,pos);
    a_path = s.substr(pos,s.size()-pos);
    return true;
  }
public:
  http(std::ostream& a_out,bool a_verbose)
  : inet_socket(a_out,a_verbose)
  {}
  virtual ~http(){}
protected:
  http(const http& a_from): inet_socket(a_from){}
  http& operator=(const http&){return *this;}
public:
  bool start(const std::string& a_host) {
    int port = 80;
    if(!connect(a_host,port,10,1)) return false;
    m_host = a_host;
    return true;
  }
  
  bool fetch(const std::string& a_req,const std::string& a_local) {
    if(m_host.empty()) return false;
    std::string answer;

    if(m_verbose) {
      m_out << "inlib::net::http::fetch :" 
            << " fetch " << sout(a_req)
            << " to be put in " << sout(a_local) << "."
            << std::endl;
    }

    //NOTE : use C str. std::string don't like CR,LF.
   {char* cmd = inlib::str_new();    
    inlib::str_cat(cmd,"GET ");
    //inlib::str_cat(cmd,"HEAD ");
    inlib::str_cat(cmd,a_req.c_str());
    inlib::str_cat(cmd," HTTP/1.1");
    inlib::str_cat(cmd,CR());
    inlib::str_cat(cmd,LF());

    inlib::str_cat(cmd,"Host: ");
    inlib::str_cat(cmd,m_host.c_str());
    inlib::str_cat(cmd,CR());
    inlib::str_cat(cmd,LF());
/*
    inlib::str_cat(cmd,"Accept: *");
    inlib::str_cat(cmd,'/');
    inlib::str_cat(cmd,"*");
    inlib::str_cat(cmd,CR());
    inlib::str_cat(cmd,LF());

    inlib::str_cat(cmd,"User-Agent: inlib");
    inlib::str_cat(cmd,CR());
    inlib::str_cat(cmd,LF());
*/
    //end :
    inlib::str_cat(cmd,CR());
    inlib::str_cat(cmd,LF());
    if(m_verbose) {
      m_out << "inlib::net::http::fetch :"
            << " send_command : " << sout(std::string(cmd)) 
            << std::endl;
    }
    if(!send_buffer(cmd,::strlen(cmd))) {
      m_out << "inlib::net::http::fetch :"
            << " send_buffer failed." 
            << std::endl;
      inlib::str_del(cmd);
      return false;
    }
    inlib::str_del(cmd);}

    std::string first_line;
    char* buffer;
    inlib::uint64 length;
    char* beg; // begin of document.
    char* end; // *end not necessary 0.
    if(!get_answer(first_line,buffer,length,beg,end)) return false;
    //WARNING : we have to delete [] buffer.

    if(m_verbose) {
      m_out << "inlib::net::http::fetch :" 
            << " received first line " << sout(first_line) << "."
            << std::endl;
    }

    if(first_line=="HTTP/1.1 200 OK"){
      if(m_verbose) {
        m_out << "inlib::net::http::fetch :" 
              << " OK."
              << std::endl;
      }

      inlib::uint64 ldoc = end-beg;
      if(m_verbose) {
        m_out << "inlib::net::http::fetch :" 
              << " document length " << ldoc
              << std::endl;
      }      

      if(!inlib::file::write_bytes(a_local,beg,(size_t)ldoc)) {
        m_out << "inlib::net::http::fetch :" 
              << "can't write local file " << inlib::sout(a_local) 
              << std::endl;
        delete [] buffer;
        return false;
      }

      if(m_verbose) {
        m_out << "inlib::net::http::fetch :" 
              << " write local file " << inlib::sout(a_local) << " ok."
              << std::endl;
      }      

      delete [] buffer;
      return true;
    }

    // troubles :
    *end = 0;    //we have added an extra char when detting data.
    std::string doc(beg); 
    delete [] buffer;
  
    if(first_line=="HTTP/1.1 301 Moved Permanently"){
      m_out << "inlib::net::http::fetch :" 
            << " received a 301 message."
            << " Document " << sout(a_req) << " moved."
            << std::endl;

      m_out << "inlib::net::http::fetch :" 
            << " full message :" << std::endl
            << doc
            << std::endl;

    } else if(first_line=="HTTP/1.1 404 Not Found"){
      m_out << "inlib::net::http::fetch :" 
            << " received a 404 message."
            << " Document " << sout(a_req) << " not found."
            << std::endl;
      m_out << "inlib::net::http::fetch :" 
            << " full message :" << std::endl
            << doc
            << std::endl;

    } else if(first_line=="HTTP/1.1 400 Bad Request"){
      m_out << "inlib::net::http::fetch :" 
            << " received a 400 message."
            << " " << sout(a_req) << " is a bad request."
            << std::endl;
      m_out << "inlib::net::http::fetch :" 
            << " full message :" << std::endl
            << doc
            << std::endl;

    } else {
      m_out << "inlib::net::http::fetch :" 
            << " first line " << sout(first_line) << " not treated."
            << std::endl;
      m_out << "inlib::net::http::fetch :" 
            << " full message :" << std::endl
            << answer
            << std::endl;

    }

    return false;
  }
protected:
  static char LF() {return 10;}
  static char CR() {return 13;}

  bool get_answer(std::string& a_first_line,char*& a_buffer,inlib::uint64& a_length,char*& a_beg,char*& a_end) {
    char* buffer;
    inlib::uint64 length;
    char* beg_doc;
    if(!get_data(buffer,length,beg_doc)) {
      m_out << "inlib::net::http::get_answer :"
            << " get_data failed." 
            << std::endl; 
      a_first_line.clear();
      a_buffer = 0;
      a_length = 0;
      a_beg = 0;
      a_end = 0;
      return false;
    }
    if(!length) {
      m_out << "inlib::net::http::get_answer :"
            << " get nothing from server." 
            << std::endl; 
      a_first_line.clear();
      a_buffer = 0;
      a_length = 0;
      a_beg = 0;
      a_end = 0;
      return false; //expect anyway something.
    }

    if(!beg_doc) {
      m_out << "inlib::net::http::get_answer :"
            << " no begin of document." 
            << std::endl; 
      a_first_line.clear();
      a_buffer = 0;
      a_length = 0;
      a_beg = 0;
      a_end = 0;
      return false; //expect anyway something.
    }

    if(m_verbose) {
      m_out << "inlib::net::http::get_answer : length " << length << std::endl;
    }

    // get first line :
   {char* end = 0;
    char* pos = buffer;
    char* pend = buffer+length;
    for(;pos!=pend;pos++) {
      if((*pos)==CR()) {
        end = pos;
        break;
      }
    }
    if(!end) {
      m_out << "inlib::net::http::get_answer :"
            << " first line end not found." 
            << std::endl; 
      delete [] buffer;
      a_first_line.clear();
      a_buffer = 0;
      a_length = 0;
      a_beg = 0;
      a_end = 0;
      return false;
    }
    char c_end = *end;
    *end = 0;
    a_first_line = buffer;
    *end = c_end;}

    /*
    // get header :
   {char* bh = beg_doc-4;
    char c_bh = *bh;
    *bh = 0;
    std::string header = buffer;
    *bh = c_bh;
    m_out << "header :" << std::endl
          << header
          << std::endl;}
    */
 
    char* end = buffer+length; //*end not necessary 0.

    if(end<beg_doc) {
      m_out << "inlib::http::fetch :" 
            << " strange answer end<beg_doc."
            << std::endl;
      delete [] buffer;
      a_first_line.clear();
      a_buffer = 0;
      a_length = 0;
      a_beg = 0;
      a_end = 0;
      return false;
    }

    a_buffer = buffer;
    a_length = length;
    a_beg = beg_doc;
    a_end = end;

    return true;
  }
  
  bool get_data(char*& a_buffer,inlib::uint64& a_length,char*& a_beg_doc){
    a_buffer = 0;
    a_length = 0;    
    a_beg_doc = 0;
    if(m_socket==(-1)) return false;

    unsigned int cl;
    bool found_cl = false;
    char* pos_search_cl = 0;

    bool found_start = false;
    char* pos_search_start = 0;

    unsigned int to_read = 4096; //sufficient to pass the header.
    unsigned int got = 0;

    unsigned int BLOCK = 65536;
    char* buf = new char[BLOCK];
    if(!buf) return false;

    while(true) {
      inlib::uint64 to_get = mn(BLOCK,to_read-got);
      if(!to_get) {
        if(m_verbose) {
          m_out << "inlib::base_socket::get_data :"
                << " finish."
                << std::endl;
        }
        if(!found_start) {
          m_out << "inlib::base_socket::get_data :"
                << " finish but begin of documen not found."
                << std::endl;
          disconnect();
          delete [] buf;
          delete [] a_buffer;
          a_buffer = 0;
          a_length = 0;    
          a_beg_doc = 0;
          return false;
        }
        delete [] buf;
        return true;
      }

      //m_out << "inlib::base_socket::get_data :"
      //      << " recv... "
      //      << std::endl;
#ifdef WIN32
      int
#else
      ssize_t
#endif
      num_char = ::recv(m_socket,buf,(size_t)to_get,0);
      if(num_char<0) {
        m_out << "inlib::base_socket::get_data :"
              << " recv : " << serror()
              << std::endl;
        disconnect();
        delete [] buf;
        delete [] a_buffer;
        a_buffer = 0;
        a_length = 0;    
        a_beg_doc = 0;
        return false;
      }
      if(num_char==0) {
        m_out << "inlib::base_socket::get_data :"
              << " recv : returned 0."
              << std::endl;

/*
        *(a_buffer+a_length) = 0;
        //printf("debug : \n%s\n",a_buffer);        
        printf("debug : xxx :\n");        
       {for(char* pos=a_buffer;*pos;pos++) {
          char c = *pos;
          if(is_printable(c)) printf("%c",c);
          else printf("%d",c);
        }
        printf("\n");}
*/

        disconnect();
        delete [] buf;
        delete [] a_buffer;
        a_buffer = 0;
        a_length = 0;    
        a_beg_doc = 0;
        return false;
      }

      if(m_verbose) {
        m_out << "inlib::base_socket::get_data :"
              << " recv : " << (int)num_char
              << std::endl;
      }
  
      if(!a_buffer) {
        //+1 so that "final end" points to something valid.
        a_buffer = new char[num_char+1];
        if(!a_buffer) {
          delete [] buf;    
          a_length = 0;    
          a_beg_doc = 0;
          return false;
        }
        ::memcpy(a_buffer,buf,num_char);
        a_length = num_char;
        pos_search_cl = a_buffer;
        pos_search_start = a_buffer;
      } else {
        unsigned long x_cl = pos_search_cl-a_buffer;
        unsigned long x_start = pos_search_start-a_buffer;

        unsigned long x_beg_doc = 0;
        if(a_beg_doc) x_beg_doc = a_beg_doc-a_buffer;

        //+1 so that "final end" points to something valid.
        char* b = new char[(size_t)(a_length+num_char+1)];
        if(!b) {
          delete [] buf;    
          delete [] a_buffer;
          a_buffer = 0;
          a_length = 0;    
          a_beg_doc = 0;
          return false;
        }
        ::memcpy(b,a_buffer,(size_t)a_length);
        ::memcpy(b+a_length,buf,num_char);
        delete [] a_buffer;
        a_buffer = b;
        a_length += num_char;

        pos_search_cl = a_buffer + x_cl;
        pos_search_start = a_buffer + x_start;

        if(a_beg_doc) a_beg_doc = a_buffer + x_beg_doc;
      }

      if(!found_cl) {
        while(true) {
          char* end = 0;
          char* pos = pos_search_cl;
          char* pend = a_buffer+a_length;
          for(;pos!=pend;pos++) {
            if((*pos)==CR()) {
              end = pos;
              break;
            }
          }
          if(!end) break;
          char c_end = *end;
          *end = 0; //transform temporarily to C string.
          pos = ::strstr(pos_search_cl,"Content-Length: ");
          if(pos) { //found it !
            if(::sscanf(pos+16,"%u",&cl)!=1) {
              m_out << "inlib::base_socket::get_data :"
                    << " sscanf error."
                    << std::endl;              
              disconnect();
              delete [] buf;
              delete [] a_buffer;
              a_buffer = 0;
              a_length = 0;    
              return false;
            }
            if(m_verbose) {
              m_out << "inlib::base_socket::get_data :"
                    << " found content-length " << cl
                    << std::endl;              
            }
            found_cl = true;
            *end = c_end;
            pos_search_start = end+1;
            break;
          }
          *end = c_end;
          pos_search_cl = end+1;
        }

      } 

      if(found_cl && !found_start) {
        // We have the document length. Search the beginning :
        if((a_length-(pos_search_start-a_buffer))>=4) {
          char* pos = pos_search_start;
          //the +1 is needed to scan end of buffer. (case of empty document).
          char* pend = a_buffer+a_length+1;
          char c1 = *pos;pos++;
          char c2 = *pos;pos++;
          char c3 = *pos;pos++;
          char c4 = *pos;pos++;
          for(;pos!=pend;pos++) {
            //{if(is_printable(c1)) printf("%c",c1);
            // else printf("%d",c1);}
            //{if(is_printable(c2)) printf("%c",c2);
            // else printf("%d",c2);}
            //{if(is_printable(c3)) printf("%c",c3);
            // else printf("%d",c3);}
            //{if(is_printable(c4)) printf("%c",c4);
            // else printf("%d",c4);}
            //printf("|\n");

            if((c1==CR())&&(c2==LF())&&(c3==CR())&&(c4==LF())){
              if(m_verbose) {
                m_out << "inlib::http::get_data :" 
                      << " begin document found."
                      << std::endl;
              }
              found_start = true;
              a_beg_doc = pos;
              break;
            }
            c1 = c2;
            c2 = c3;
            c3 = c4;
            c4 = *pos;
          }
        }
      }

      if(found_start) {

        // have cl and found beg_doc !
        to_read = (a_beg_doc-a_buffer)+cl;

      }

      got += num_char;
    }
  }

protected:
  std::string m_host;
};

}}

#endif