/Users/barrand/private/dev/softinex/old/inexlib-1.2/inlib/inlib/wroot/file

Go to the documentation of this file.

// Copyright (C) 2010, Guy Barrand. All rights reserved.
// See the file inlib.license for terms.

#ifndef inlib_wroot_file
#define inlib_wroot_file

// inheritance :
#include "ifile"

#include "directory"

#include "infos"

#include "../platform"

#include "../path"
        
#include <map>

#include <fcntl.h>
#include <errno.h>
#include <sys/stat.h>

#if defined(WIN32) && !defined(__GNUC__)
#include <direct.h>
#define ssize_t int
#include <io.h>
// disable the warning about the usage of "this" in the constructor.
#pragma warning(disable:4355)
#endif

namespace inlib {
namespace wroot {

class file : public virtual ifile {
  static const std::string& s_class() {
    static const std::string s_v("inlib::wroot::file");
    return s_v;
  }
  static int not_open() {return -1;}
  static uint32 kBegin() {return 64;}
  static uint32 START_BIG_FILE() {return 2000000000;}
public: //ifile
  virtual bool verbose() const {return m_verbose;}
  virtual std::ostream& out() {return m_out;}

  virtual bool byte_swap() const {return is_little_endian();}
  virtual bool set_pos(seek a_offset = 0,from a_from = begin){
    int whence = 0;
    switch(a_from) {
    case begin:
      whence = SEEK_SET;
      break;
    case current:
      whence = SEEK_CUR;
      break;
    case end:
      whence = SEEK_END;
      break;
    }

#if defined(__linux__) && (__GLIBC__ == 2) && (__GLIBC_MINOR__ >= 2)
    if (::lseek64(m_file, a_offset, whence) < 0) {
#elif defined(WIN32)
    if (::_lseeki64(m_file, a_offset, whence) < 0) {
#else
    if (::lseek(m_file, a_offset, whence) < 0) {
#endif
      m_out << "inlib::wroot::file::set_pos :"
            << " cannot set position " << a_offset 
            << " in file " << sout(m_path) << "."
            << std::endl;
      return false;
    }
    return true;
  }

  virtual seek END() const {return m_END;}
  virtual void set_END(seek a_end){m_END = a_end;}

  virtual bool write_buffer(const char* a_buffer,uint32 a_length) {
    // Write a buffer to the file. This is the basic low level write operation.
    ssize_t siz;
    while ((siz = ::write(m_file,a_buffer,a_length)) < 0 && 
            error_number() == EINTR) reset_error_number();

    if(siz < 0) {
      m_out << "inlib::wroot::file::write_buffer :"
            << " error writing to file " << sout(m_path) << "."
            << std::endl;
      return false;
    }
    if(siz!=(ssize_t)a_length) {
      m_out << "inlib::wroot::file::write_buffer :"
           << "error writing all requested bytes to file " << sout(m_path)
           << ", wrote " << long2s(siz) << " of " << a_length
           << std::endl;
      return false;
    }
    //m_bytes_write  += siz;
    return true;
  }

  virtual uint32 version() const {
    // Return version id as an integer, i.e. "2.22/04" -> 22204.
    static const uint32 ROOT_MAJOR_VERSION = 4;
    static const uint32 ROOT_MINOR_VERSION = 0;
    static const uint32 ROOT_PATCH_VERSION = 0;
    return 
      10000 * ROOT_MAJOR_VERSION + 
      100 * ROOT_MINOR_VERSION + 
      ROOT_PATCH_VERSION;
  }

  virtual bool synchronize(){
    // Synchornize a file's in-core and on-disk states.
#ifdef WIN32
    return true;
#else
    if (::fsync(m_file) < 0) {
      m_out << "inlib::wroot::file::synchronize :"
            << " error flushing file " << sout(m_path) << "."
            << std::endl;
      return false;
    } 
    return true;
#endif
  }

  virtual bool ziper(char a_key,zip_func& a_func) const {
    std::map<char,zip_func>::const_iterator it = m_zipers.find(a_key);
    if(it==m_zipers.end()) {
      a_func = 0;
      return false;
    }
    a_func = (*it).second;
    return true;
  }
  virtual uint32 compression() const {return m_compress;}
  virtual void compress_buffer(const buffer& a_buffer,
                                    char*& a_kbuf,uint32& a_klen,bool& a_kdel){
    //NOTE : if(kdelete) delete [] kbuf;

    a_kbuf = 0;
    a_klen = 0;
    a_kdel = false;

    uint32 nbytes = a_buffer.length();
    uint32 cxlevel = m_compress;
    if(cxlevel && (nbytes>256)) {
      ifile::zip_func func;
      if(!ziper('Z',func)) {
        //m_out << "inlib::wroot::directory::write_object :" 
        //      << " zlib ziper not found."
        //      << std::endl;
        a_kbuf = (char*)a_buffer.buf();
        a_klen = a_buffer.length();
        a_kdel = false;
      } else {
        const uint32 kMAXBUF = 0xffffff;
        const uint32 HDRSIZE = 9;
        uint32 nbuffers = nbytes/kMAXBUF;
        uint32 buflen = nbytes+HDRSIZE*(nbuffers+1); 
        a_kbuf = new char[buflen];
        a_kdel = true;
        char* src = (char*)a_buffer.buf();
        char* tgt = a_kbuf;
        uint32 nzip = 0;
        for(uint32 i=0;i<=nbuffers;i++) {
          uint32 bufmax = ((i == nbuffers) ? nbytes - nzip : kMAXBUF);
          uint32 nout;
          if(!zip(m_out,func,cxlevel,bufmax,src,bufmax,tgt,nout)) {
            delete [] a_kbuf;
            a_kbuf = (char*)a_buffer.buf();
            a_klen = a_buffer.length();
            a_kdel = false;
            break;
          }
          tgt += nout; //nout includes HDRSIZE
          a_klen += nout;
          src += kMAXBUF;
          nzip += kMAXBUF;
        }    
        //::printf("debug : compress : end : %u %u\n",nbytes,klen);
      }
    } else {
      a_kbuf = (char*)a_buffer.buf();
      a_klen = a_buffer.length();
      a_kdel = false;
    }
  }
public:
  file(std::ostream& a_out,
              const std::string& a_path,
              bool a_verbose = false)
  :m_out(a_out)
  ,m_path(a_path)
  ,m_verbose(a_verbose)
  ,m_file(not_open())
  //,m_bytes_write(0)
  ,m_root_directory(*this,nosuffix(a_path),m_title)
  // begin of record :
  ,m_version(0)
  ,m_BEGIN(0)
  ,m_END(0)
  ,m_seek_free(0)
  ,m_nbytes_free(0)
  ,m_nbytes_name(0)
  ,m_units(4)
  ,m_compress(1)
  ,m_seek_info(0)
  ,m_nbytes_info(0)
  {
#ifdef INLIB_MEM
    mem::increment(s_class().c_str());
#endif

    m_version = version();

    if(access_path(m_path,kFileExists)) unlink(m_path);

    if(!m_root_directory.is_valid()) {
      m_out << "inlib::wroot::file::file :"
            << " " << sout(m_path) << " root directory badly created."
            << std::endl;
      return;
    }

    m_file = _open(a_path.c_str(),
#ifdef WIN32
                               O_RDWR | O_CREAT | O_BINARY,S_IREAD | S_IWRITE
#else
                               O_RDWR | O_CREAT,0644
#endif
    );
    if(m_file==not_open()) {
      m_out << "inlib::wroot::file::file :"
            << " can't open " << sout(a_path) << "."
            << std::endl;
      return;
    }

    //initialize :
    m_BEGIN = kBegin();  // First used word in file following the file header.
    m_END = m_BEGIN;   // Pointer to end of file.

    // Write Directory info :
    uint32 namelen = 
      key::std_string_record_size(m_path) + 
      key::std_string_record_size(m_title);
    uint32 nbytes = namelen + m_root_directory.record_size();
    wroot::key key(*this,0,m_path,m_title,"TFile",nbytes); //set m_END.

    // m_nbytes_name = start point of directory info from key head.
    m_nbytes_name = key.key_length() + namelen; 
    m_root_directory.set_nbytes_name(m_nbytes_name);
    m_root_directory.set_seek_directory(key.seek_key()); //at EOF.

    //the below write 45 bytes at BOF (Begin Of File).
    if(!write_header()) { //need m_nbytes_name, m_END after key written.
      m_out << "inlib::wroot::file::file :"
            << " can't write file header."
            << std::endl;
      return;
    }

   {char* pos = key.data_buffer();
    wbuf wb(m_out,byte_swap(),key.eob(),pos);
    if(!wb.write(m_path)) return;
    if(!wb.write(m_title)) return;
    if(!m_root_directory.to_buffer(wb)) return;}

    if(m_verbose) {
      m_out << "inlib::wroot::file::file :"
            << " write key ("
            << namelen 
            << ", " 
            << m_root_directory.record_size()
            << ", " 
            << nbytes
            << ", "
            << m_nbytes_name
            << ", "
            << key.seek_key() 
            << ")."
            << std::endl;
    }

    key.set_cycle(1);
    if(!key.write_self()) {
      m_out << "inlib::wroot::file::file :"
            << " key.write_self() failed."
            << std::endl;
      return;
    }

    //the below write at kBegin + nbytes.
    //64+52
    uint32 n;
    if(!key.write_file(n)) {
      m_out << "inlib::wroot::file::file :"
            << " can't write key in file."
            << std::endl;
      return;
    }
    //::printf("debug : file::file : write key : %d\n",n);

  }
  virtual ~file() {
    close();
#ifdef INLIB_MEM
    mem::decrement(s_class().c_str());
#endif
  }
protected:
  file(const file& a_from)
  :ifile(a_from)
  ,m_out(a_from.m_out)
  ,m_root_directory(*this)
  { 
#ifdef INLIB_MEM
    mem::increment(s_class().c_str());
#endif
  }
  file& operator=(const file&){return *this;}
public:
  void set_compression(uint32 a_level) {
    // level = 0 objects written to this file will not be compressed.
    // level = 1 minimal compression level but fast.
    // ....
    // level = 9 maximal compression level but slow.
    m_compress = a_level;
    if(m_compress>9) m_compress = 9;
  }

  void close() {
    if(m_file==not_open()) return;
    m_root_directory.close();
    ::close(m_file);
    m_file = not_open();
  }

  directory& dir() {return m_root_directory;}
  const directory& dir() const {return m_root_directory;}

  bool write(uint32& a_nbytes){
    // Write memory objects to this file :
    //  Loop on all objects in m_root_directory (including subdirectories).
    //  A new key is created in the directories m_keys linked list
    //  for each object.
    //  The list of keys is then saved on the file (via write_keys)
    //  as a single data record.
    //  The directory header info is rewritten on the directory header record.
    //  //The linked list of FREE segments is written.
    //  The file header is written (bytes 1->m_BEGIN).
    a_nbytes = 0;

    if(m_verbose) {
      m_out << "inlib::wroot::file::write :"
            << " writing Name=" << sout(m_path)
            << " Title=" << sout(m_title) << "."
            << std::endl;
    }

    uint32 nbytes;
    if(!m_root_directory.write(nbytes)) return false; // Write directory tree

    if(!write_streamer_infos()) {
      m_out << "inlib::wroot::file::write :"
            << " write_streamer_infos failed."
            << std::endl;
      return false;
    }

    //if(!writeFreeSegments()) return false; // Write free segments.

    if(!write_header()) { //write 45 bytes at BOF.
      m_out << "inlib::wroot::file::write :"
            << " can't write file header."
            << std::endl;
      return false;
    }

    a_nbytes = nbytes;
    return true;
  }

  bool add_ziper(char a_key,zip_func a_func){
    std::map<char,zip_func>::iterator it = m_zipers.find(a_key);
    if(it!=m_zipers.end()) {
      //(*it).second = a_func; //override ?
      return false;
    } else {
      m_zipers[a_key] = a_func;
      return true;
    }
  }
protected:
  enum EAccessMode {
    kFileExists        = 0,
    kExecutePermission = 1,
    kWritePermission   = 2,
    kReadPermission    = 4
  };
  static bool access_path(const std::string& a_path,EAccessMode a_mode){
    // Returns true if one can access a file using the specified access mode.
    // Mode is the same as for the WinNT access(2) function.
#ifdef WIN32
    return (::_access(a_path.c_str(),a_mode) == 0) ? true : false;
#else
    return (::access(a_path.c_str(),a_mode) == 0) ? true : false;
#endif
  }
  static bool unlink(const std::string& a_path){
    // Unlink, i.e. remove, a file or directory. Returns true when succesfull,
    // false in case of failure.
    struct stat finfo;
    if (::stat(a_path.c_str(),&finfo) < 0) return false;
#ifdef WIN32 
    if (finfo.st_mode & S_IFDIR)
      return (::_rmdir(a_path.c_str())==-1 ? false : true);
    else
      return (::unlink(a_path.c_str())==-1 ? false : true);
#else
    if (S_ISDIR(finfo.st_mode))
      return (::rmdir(a_path.c_str())==-1 ? false : true);
    else
      return (::unlink(a_path.c_str())==-1 ? false : true);
#endif
  }

  static int _open(const char* a_name,int a_flags,unsigned int a_mode) {
#if defined(__linux__) && (__GLIBC__ == 2) && (__GLIBC_MINOR__ >= 2)
     return ::open64(a_name,a_flags,a_mode);
#else
     return ::open(a_name,a_flags,a_mode);
#endif
  }
  static std::string sout(const std::string& a_string) {
    return std::string("\"")+a_string+"\"";
  }
  bool write_header() {
    const char root[] = "root";
    //char psave[kBegin()];
    char psave[128];
    const char* eob = psave + kBegin();
    char* pos = psave;
    ::memcpy(pos,root,4); pos += 4;
    uint32 version = m_version;
    if((m_END>START_BIG_FILE())        ||
       (m_seek_free>START_BIG_FILE())  ||
       (m_seek_info>START_BIG_FILE())  ){
      version += 1000000;
      m_units = 8;
    }
    wbuf wb(m_out,byte_swap(),eob,pos);
    if(!wb.write(version)) return false;
    if(!wb.write((seek32)m_BEGIN)) return false;
    if(version>1000000) {
      if(!wb.write(m_END)) return false;
      if(!wb.write(m_seek_free)) return false;
    } else {
      if(!wb.write((seek32)m_END)) return false;
      if(!wb.write((seek32)m_seek_free)) return false;
    }
    if(!wb.write(m_nbytes_free)) return false;
    //int nfree  = fFreeSegments.size();
    uint32 nfree  = 0; //FIXME
    if(!wb.write(nfree)) return false;
    if(!wb.write(m_nbytes_name)) return false;
    if(!wb.write(m_units)) return false;
    if(!wb.write(m_compress)) return false;
    if(version>1000000) {
      if(!wb.write(m_seek_info)) return false;
    } else {
      if(!wb.write((seek32)m_seek_info)) return false;
    }
    if(!wb.write(m_nbytes_info)) return false;
    if(!set_pos()) return false; //BOF
    uint32 nbytes = pos - psave;
    //::printf("debug : write_header : %d\n",nbytes);
    if(!write_buffer(psave,nbytes)) return false;
    if(!synchronize()) return false;
    return true;
  }

  bool write_streamer_infos() {
    List<StreamerInfo> sinfos;
    fill_infos(sinfos,m_out);

    if(sinfos.empty()) return false;

    buffer bref(m_out,byte_swap(),256);

    if(!sinfos.stream(bref)) {
      m_out << "inlib::wroot::file::write_streamer_infos :" 
            << " cannot stream List<StreamerInfo>."
            << std::endl;
      return false;
    }
    uint32 nbytes = bref.length();

    wroot::key key(*this,
                   m_root_directory.seek_directory(),
                   "StreamerInfo","",
                   sinfos.store_cls(),
                   nbytes); //set m_END
    if(!key.seek_key()) return false;

    if(!bref.displace_mapped(key.key_length())) return false;

    ::memcpy(key.data_buffer(),bref.buf(),nbytes);

    //key.set_cycle(1);
    if(!key.write_self()) {
      m_out << "inlib::wroot::file::write_streamer_infos :" 
            << " key.write_self() failed."
            << std::endl;
      return false;
    }

    m_seek_info = key.seek_key();
    m_nbytes_info = key.number_of_bytes();
    //FIXME sumBuffer(key.objectSize());


    uint32 n;
    if(!key.write_file(n)) return false;
    if(!n) return false;

    return true;
  }

  static bool zip(std::ostream& a_out,
                         ifile::zip_func a_func,
                         int a_level,
                         uint32 a_srcsize,char* a_src,
                         uint32 a_tgtsize,char* a_tgt,
                         uint32& a_irep){

    // from Rio/Bits/R__zip using zlib.

    const uint32 HDRSIZE = 9;

    if(a_tgtsize<HDRSIZE) {
      a_out << "inlib::rroot::directory::zip :"
            << " target buffer too small."
            << std::endl;
      a_irep = 0; 
      return false;
    }
    if(a_srcsize>0xffffff) {
      a_out << "inlib::rroot::directory::zip :"
            << " source buffer too big."
            << std::endl;
      a_irep = 0; 
      return false;
    }

    uint32 out_size;
    if(!a_func(a_out,a_level,
               a_srcsize,a_src,
               a_tgtsize,a_tgt+HDRSIZE,
               out_size)) {
      a_out << "inlib::rroot::directory::zip :"
            << " zipper failed."
            << std::endl;
      a_irep = 0; 
      return false;
    }
    if((HDRSIZE+out_size)>a_tgtsize) {
      a_out << "inlib::rroot::directory::zip :"
            << " target buffer overflow."
            << std::endl;
      a_irep = 0; 
      return false;
    }

    // HEADER :
    a_tgt[0] = 'Z'; // Signature ZLib
    a_tgt[1] = 'L';
    a_tgt[2] = 8; //DEFLATE
 
    a_tgt[3] = (char)(out_size & 0xff);
    a_tgt[4] = (char)((out_size >> 8) & 0xff);
    a_tgt[5] = (char)((out_size >> 16) & 0xff);
 
    a_tgt[6] = (char)(a_srcsize & 0xff);
    a_tgt[7] = (char)((a_srcsize >> 8) & 0xff);
    a_tgt[8] = (char)((a_srcsize >> 16) & 0xff);

    a_irep = HDRSIZE+out_size;

    return true;
  }

#if defined(__sun) && !defined(__linux__) && (__SUNPRO_CC > 0x420)
  int error_number() {return ::errno;}
  void reset_error_number() {::errno = 0;}
#else
  int error_number() {return errno;}
  void reset_error_number() {errno = 0;}
#endif

protected:
  std::ostream& m_out;
  std::string m_path;
  bool m_verbose;
  int m_file;
  //uint64 m_bytes_write; //Number of bytes write in this file
  std::string m_title; //must be before the below.
  directory m_root_directory;
  std::map<char,zip_func> m_zipers;
  // begin of record :
  // "root"
  uint32 m_version;       //File format version
  seek m_BEGIN;           //First used byte in file
  seek m_END;             //Last used byte in file
  seek m_seek_free;       //Location on disk of free segments structure
  uint32 m_nbytes_free;   //Number of bytes for free segments structure
  //int nfree
  uint32 m_nbytes_name;   //Number of bytes in TNamed at creation time
  char m_units;           //Number of bytes for file pointers
  uint32 m_compress;      //(=1 file is compressed, 0 otherwise)
  seek m_seek_info;       //Location on disk of StreamerInfo record
  uint32 m_nbytes_info;   //Number of bytes for StreamerInfo record
};


}}

#endif

//doc
//
//  A ROOT file is a suite of consecutive data records with the following
//    format (see also the TKey class);
// TKey ---------------------
//      byte 1->4  Nbytes    = Length of compressed object (in bytes)
//           5->6  Version   = TKey version identifier
//           7->10 ObjLen    = Length of uncompressed object
//          11->14 Datime    = Date and time when object was written to file
//          15->16 KeyLen    = Length of the key structure (in bytes)
//          17->18 Cycle     = Cycle of key
//          19->22 SeekKey   = Pointer to record itself (consistency check)
//          23->26 SeekPdir  = Pointer to directory header
//          27->27 lname     = Number of bytes in the class name
//          28->.. ClassName = Object Class Name
//          ..->.. lname     = Number of bytes in the object name
//          ..->.. Name      = lName bytes with the name of the object
//          ..->.. lTitle    = Number of bytes in the object title
//          ..->.. Title     = Title of the object
//          -----> DATA      = Data bytes associated to the object
//
//  The first data record starts at byte fBEGIN (currently set to kBegin)
//  Bytes 1->kBegin contain the file description:
//       byte  1->4  "root"      = Root file identifier
//             5->8  fVersion    = File format version
//             9->12 fBEGIN      = Pointer to first data record
//            13->16 fEND        = Pointer to first free word at the EOF
//            17->20 fSeekFree   = Pointer to FREE data record
//            21->24 fNbytesFree = Number of bytes in FREE data record
//            25->28 nfree       = Number of free data records
//            29->32 fNbytesName = Number of bytes in TNamed at creation time
//            33->33 fUnits      = Number of bytes for file pointers
//            34->37 fCompress   = Zip compression level
//