source: Sophya/trunk/SophyaLib/HiStats/basedtable.cc@ 3712

#include "basedtable.h"
#include <ctype.h>
#include "pexceptions.h"
#include "thsafeop.h"

namespace SOPHYA {

/*!
   \class DataTableRow
   \ingroup HiStats
   This class is intented to be used with datatable classes
   (inheriting from BaseDataTable) for representing a row (line)
   of the table. 
*/
DataTableRow::DataTableRow( vector<string> const& colnames )
{
  if (colnames.size() < 1) 
    throw ParmError("DataTableRow::DataTableRow(vector<string>& cn) cn.size()==0 ");
  size_ = colnames.size();
  mtv_ = new MuTyV[ size_ ];
  for(sa_size_t k=0; k<size_; k++) 
    nm2idx_[colnames[k]] = k;
}

DataTableRow::DataTableRow(DataTableRow const & a )
{
  size_ = a.size_;
  mtv_ = new MuTyV[ size_ ];
  nm2idx_ = a.nm2idx_;
}

DataTableRow::~DataTableRow()
{
  delete[] mtv_;
}

MuTyV DataTableRow::get(string const& colname) const
{
  map<string, sa_size_t>::const_iterator it = nm2idx_.find(colname);
  if (it == nm2idx_.end()) { 
    string msg = "DataTableRow::get( " ;
    msg += colname;    msg += " ) Not found column name";
    throw NotFoundExc(msg);
  }
  return mtv_[(*it).second];
}

MuTyV& DataTableRow::get(string const& colname) 
{
  map<string, sa_size_t>::const_iterator it = nm2idx_.find(colname);
  if (it == nm2idx_.end()) { 
    string msg = "DataTableRow::get( " ;
    msg += colname;    msg += " ) const - Not found column name";
    throw NotFoundExc(msg);
  }
  return mtv_[(*it).second];
}

ostream&   DataTableRow::Print(ostream& os) const
{
  for(sa_size_t k=0; k<size_; k++) 
    os << (string)mtv_[k] << " ";
  return os;
}

/*!
   \class BaseDataTable
   \ingroup HiStats
   Base class for data tables. Each line represent a record
   and the column contains a given data type.
   \warning 
   Thread safe fill operation can be activated using the method SetThreadSafe()
   Default mode is NON thread-safe fill.   
*/

/*!
  if fgl == true , return LongForm string
  \verbatim
  ----------------------------------------------  
  FieldType         ShortForm     LongForm
  ----------------------------------------------  
  IntegerField          I         Integer
  LongField             L         Long Integer
  FloatField            F         Float
  DoubleField           D         Double
  ComplexField         Zx4        Complex
  DoubleComplexField   Zx8        DoubleComplex
  StringField           S         String
  DateTimeField         T         DateTime
  ----------------------------------------------  
  \endverbatim
*/
string BaseDataTable::ColTypeToString(FieldType ft, bool fgl)
{
  string rs;
  switch (ft) {
  case IntegerField :
    if (fgl) rs = "Integer";
    else rs = "I";
    break;
  case LongField :
    if (fgl) rs = "Long Integer";
    else rs = "L";
    break;
  case FloatField :
    if (fgl) rs = "Float";
    else rs = "F";
    break;
  case DoubleField :
    if (fgl) rs = "Double";
    else rs = "D";
    break;
  case ComplexField :
    if (fgl) rs = "Complex";
    else rs = "Zx4";    
    break;
  case DoubleComplexField :
    if (fgl) rs = "DoubleComplex";
    else rs = "Zx8";
    break;    
  case StringField :
    if (fgl) rs = "String";
    else rs = "S";
    break;
  case DateTimeField :
    if (fgl) rs = "DateTime";
    else rs = "T";
    break;
  default:
    rs = "??";
    break;
  }
  return rs;
}

BaseDataTable::FieldType BaseDataTable::StringToColType(string const & sctyp)
{
  if ( (sctyp == "Integer") || (sctyp == "I") )  return IntegerField;
  else if ( (sctyp == "Long Integer") || (sctyp == "L") )  return LongField;
  else if ( (sctyp == "Float") || (sctyp == "F") )  return FloatField;
  else if ( (sctyp == "Double") || (sctyp == "D") )  return DoubleField;
  else if ( (sctyp == "Complex") || (sctyp == "Zx4") )  return ComplexField;
  else if ( (sctyp == "DoubleComplex") || (sctyp == "Zx8") || (sctyp == "Z") )  
    return DoubleComplexField;
  else if ( (sctyp == "String") || (sctyp == "S") )  return StringField;
  else if ( (sctyp == "DateTime") || (sctyp == "T") )  return DateTimeField;
  else return DoubleField;
}

/* Constructeur */
BaseDataTable::BaseDataTable(sa_size_t segsz)
{
  mNEnt = 0;
  mSegSz = (segsz > 0) ? segsz : 16;
  mNSeg = 0;
  mVarD = NULL;
  mVarMTV = NULL;
  mInfo = NULL;
  mThS = NULL;
}

BaseDataTable::~BaseDataTable()
{
  if (mVarD) delete[] mVarD;
  if (mVarMTV) delete[] mVarMTV;
  if (mInfo) delete mInfo;
  if (mThS) delete mThS;
}

/*!
  \brief Activate or deactivate thread-safe \b AddRow() operations

  If fg==true, create an ThSafeOp object in order to insure atomic AddRow()
  and GetRow()/GetColumn() operations. if fg==false, the ThSafeOp object (mThS) 
  of the target DataTable is destroyed.

  When activated, the following operations are thread-safe : 

  - AddRow(const r_8* data)
  - AddRow(const MuTyV* data)
  - AddRow(DataTableRow const& data)
  - GetRow(sa_size_t n, DataTableRow& row)
  - GetRow(sa_size_t n, MuTyV* mtvp)
  - GetColumnD(sa_size_t k)

  \warning The default AddRow() operation mode for DataTables is NOT thread-safe. 
  Please note also that the thread-safety state is NOt saved to PPF (or FITS) streams.
*/
void BaseDataTable::SetThreadSafe(bool fg) 
{
  if (fg) {
    if (mThS) return;
    else mThS = new ThSafeOp();
  }
  else {
    if (mThS) delete mThS;
    mThS = NULL;
  }
}


/*! \cond
  \class DT_TSOP_SYNC 
  \ingroup HiStats 
  Classe utilitaire pour faciliter la gestion de lock pour 
  operations thread-safe BaseDataTable
*/
class DT_TSOP_SYNC {
 public:
  explicit DT_TSOP_SYNC(ThSafeOp* ths) 
  { 
    ths_ = ths; 
    if (ths_) ths_->lock(); 
  }
  ~DT_TSOP_SYNC() 
  {
    if (ths_) ths_->unlock(); 
  }
  inline ThSafeOp* NOp() { return ths_; }
 protected:
  ThSafeOp* ths_;
};
/*! \endcond */

sa_size_t BaseDataTable::CopyStructure(BaseDataTable const & a)
{
  if (NVar() > 0) 
    throw ParmError("BaseDataTable::CopyStructure() Table already has columns");
  if (a.NVar() == 0) {
    cout << "BaseDataTable::CopyStructure(a)/Warning Table a is not initialized" << endl;
    return 0;
  }
  for (size_t kk=0; kk<a.mNames.size(); kk++) 
    AddColumn(a.mNames[kk].type, a.mNames[kk].nom);
  return NVar();
}

//! return true is same structure
bool BaseDataTable::CompareStructure(BaseDataTable const & a)
{
  if (NVar() != a.NVar())  return false;
  for (size_t kk=0; kk<mNames.size(); kk++) 
    if ( (mNames[kk].type != a.mNames[kk].type) || 
         (mNames[kk].nom != a.mNames[kk].nom) ) return false;
  return true;
}

//! Check the validity of column name \b cnom - throw exception.
bool BaseDataTable::CheckColName(string const & cnom)
{
  size_t l,k;
  l = cnom.length();
  if (l < 1)  
    throw ParmError("BaseDataTable::CheckColName() zero length column name");
  if (!isalpha(cnom[0]))  
    throw ParmError("BaseDataTable::CheckColName() first character not alphabetical");
  for(k=1; k<l; k++) 
    if ((!isalnum(cnom[k])) && (cnom[k] != '_'))  
      throw ParmError("BaseDataTable::CheckColName() Non alphanumeric char in name");
  for(k=0; k<mNames.size(); k++)
    if (cnom == mNames[k].nom) 
      throw ParmError("BaseDataTable::CheckColName() already existing name");
  return true;
}

/*! 
  \brief Check  the validity of column name \b cnom and correct it if necessary
  return true if \b cnom has been changed
*/
bool BaseDataTable::CheckCorrectColName(string& cnom)
{
  size_t l,k;
  l = cnom.length();
  string ssn;
  MuTyV mvv = (NCols()+1);
  mvv.Convert(ssn);
  bool fgrc = false;
  if (l < 1) { cnom = "C"+ssn;   fgrc = true; }
  if (!isalpha(cnom[0]))  { cnom = "C"+cnom;  fgrc = true; }
  for(k=1; k<l; k++) 
    if ((!isalnum(cnom[k])) && (cnom[k] != '_'))  { 
      cnom[k] = '_';  fgrc = true;
    }
  bool fgdouble = true;
  while (fgdouble) {
    fgdouble = false;
    for(k=0; k<mNames.size(); k++)
      if (cnom == mNames[k].nom) {
        fgdouble = true;  break;
      }
    if (fgdouble) { cnom = cnom+ssn; fgrc = true; } 
  }
  return fgrc;
}

// Retourne une structure
/*!
 The returned BaseDataTable object can be used for subsequent call to 
 AddRow() or GetRow() methods.
 Generate an exception if called for a table with no columns
*/
DataTableRow BaseDataTable::EmptyRow()
{
  if (NCols() == 0) 
    throw ParmError("BaseDataTable::EmptyRow() Table has no column !");
  vector<string> nms;
  for(sa_size_t k=0; k<NVar(); k++)  nms.push_back(mNames[k].nom);
  DataTableRow row(nms);
  return row;
}

// 
// A quel index correspond mon nom ? 
// 
sa_size_t BaseDataTable::IndexNom(char const* nom) const 
{
  for(sa_size_t k=0; k<NVar(); k++) 
    if ( mNames[k].nom == nom )      return k;
  return -1;
  // Reza:Avril 2005 : PINtuple se base sur le renvoi de -1 et pas d'une exception
  //  throw NotFoundExc("BaseDataTable::IndexNom() : column name not found ");
}

string BaseDataTable::NomIndex(sa_size_t k) const 
{
  if ((k < 0) || (k >= NVar())) 
    throw RangeCheckError("BaseDataTable::NomIndex() out of range column index k");
  return mNames[k].nom ; 
}

//! Adds a row (or line) to the table with r_8* inout data
/*! 
  The data to be added is provided as an array (vector) of double (r_8).
  The necessary data conversion is performed, depending on each 
  column's data typeconverted to the data type. 
  Return the new number of table rows (lines / entries)
  \param data : Data for each cell of the row to be appended 
  (data[k] k=0..NbColumns()) 
*/
sa_size_t BaseDataTable::AddRow(const r_8* data)
{
  DT_TSOP_SYNC dttss(mThS); dttss.NOp();  // Thread-safe operation 

  if (NVar() == 0) 
    throw ParmError("BaseDataTable::AddRow(const r_8*) Table has no column !");
  if (NEntry() == SegmentSize()*NbSegments())  Extend();
  sa_size_t n = NEntry();
  sa_size_t bid = n/mSegSz;
  sa_size_t off = n%mSegSz;
  for(sa_size_t k=0; k<mIColsP.size(); k++) 
    mIColsP[k]->GetSegment(bid)[off] = (int_4)data[mIColIdx[k]];
  for(sa_size_t k=0; k<mLColsP.size(); k++) 
    mLColsP[k]->GetSegment(bid)[off] = (int_8)data[mLColIdx[k]];
  for(sa_size_t k=0; k<mFColsP.size(); k++) 
    mFColsP[k]->GetSegment(bid)[off] = (r_4)data[mFColIdx[k]];
  for(sa_size_t k=0; k<mDColsP.size(); k++) 
    mDColsP[k]->GetSegment(bid)[off] = data[mDColIdx[k]];
  for(sa_size_t k=0; k<mYColsP.size(); k++) 
    mYColsP[k]->GetSegment(bid)[off] = complex<r_4>(data[mYColIdx[k]],0.);
  for(sa_size_t k=0; k<mZColsP.size(); k++) 
    mZColsP[k]->GetSegment(bid)[off] = complex<r_8>(data[mZColIdx[k]],0.);
  for(sa_size_t k=0; k<mSColsP.size(); k++) 
    mSColsP[k]->GetSegment(bid)[off] = (string)MuTyV(data[mSColIdx[k]]);

  mNEnt++;
  return mNEnt;
}

//! Adds a row (or line) to the table with input data as an array of MuTyV
/*! 
  The data to be added is provided as an array (vector) of MuTyV.
  The MuTyV class conversion operators are used to match against each 
  cell data type.
  Return the new number of table rows (lines / entries)
  \param data : Data (MuTyV*) for each cell of the row to be appended 
  (data[k] k=0..NbColumns()) 
*/
sa_size_t BaseDataTable::AddRow(const MuTyV* data)
{
  DT_TSOP_SYNC dttss(mThS); dttss.NOp();  // Thread-safe operation 

  if (NVar() == 0) 
    throw ParmError("BaseDataTable::AddRow(const MuTyV*) Table has no column !");
  if (NEntry() == SegmentSize()*NbSegments())  Extend();
  sa_size_t n = NEntry();
  sa_size_t bid = n/mSegSz;
  sa_size_t off = n%mSegSz;
  for(sa_size_t k=0; k<mIColsP.size(); k++) 
    mIColsP[k]->GetSegment(bid)[off] = (int_4)data[mIColIdx[k]];
  for(sa_size_t k=0; k<mLColsP.size(); k++) 
    mLColsP[k]->GetSegment(bid)[off] = (int_8)data[mLColIdx[k]];
  for(sa_size_t k=0; k<mSColsP.size(); k++) 
    mSColsP[k]->GetSegment(bid)[off] = (string)data[mSColIdx[k]];
  for(sa_size_t k=0; k<mFColsP.size(); k++) 
    mFColsP[k]->GetSegment(bid)[off] = (r_4)data[mFColIdx[k]];
  for(sa_size_t k=0; k<mDColsP.size(); k++) 
    mDColsP[k]->GetSegment(bid)[off] = (r_8)data[mDColIdx[k]];
  for(sa_size_t k=0; k<mYColsP.size(); k++) 
    mYColsP[k]->GetSegment(bid)[off] = 
      complex<r_4>(data[mYColIdx[k]].GetRealPart(), data[mYColIdx[k]].GetImagPart());
  for(sa_size_t k=0; k<mZColsP.size(); k++) 
    mZColsP[k]->GetSegment(bid)[off] = 
      complex<r_8>(data[mZColIdx[k]].GetRealPart(), data[mZColIdx[k]].GetImagPart());

  mNEnt++;
  return mNEnt;
}
//! Adds a row (or line) to the table with input data as DataTableRow object
/*! 
  The internal MuTyV array of the object contains the data and the 
  MuTyV class conversion operators are used to match against each 
  cell data type.
  Only the size of the input data object is checked.
  Return the new number of table rows (lines / entries)
  \param data : Data  for each cell of the row to be appended 
  (data[k] k=0..NbColumns()) 
*/
sa_size_t BaseDataTable::AddRow(DataTableRow const& data)
{
  if ( data.Size() != NCols() )
    throw SzMismatchError(" BaseDataTable::AddRow() - data.Size() != NCols() ");
  return AddRow(data.MTVPtr());
}

/*! 
  Extends the table (in the row direction). This method is called automatically when needed.
*/
sa_size_t BaseDataTable::Extend()
{
  for(sa_size_t k=0; k<mIColsP.size(); k++) 
    mIColsP[k]->Extend();
  for(sa_size_t k=0; k<mLColsP.size(); k++) 
    mLColsP[k]->Extend();
  for(sa_size_t k=0; k<mFColsP.size(); k++) 
    mFColsP[k]->Extend();
  for(sa_size_t k=0; k<mDColsP.size(); k++)     
    mDColsP[k]->Extend();
  for(sa_size_t k=0; k<mYColsP.size(); k++) 
    mYColsP[k]->Extend();
  for(sa_size_t k=0; k<mZColsP.size(); k++) 
    mZColsP[k]->Extend();
  for(sa_size_t k=0; k<mSColsP.size(); k++) 
    mSColsP[k]->Extend();
  mNSeg++;
  return mNSeg;
}

/*!
  Fills the input \b row object with the content of row \b n.
  Return a reference to the input \b row object. 
  Generate an exception if the input \b row object has the wrong size.
*/
DataTableRow& BaseDataTable::GetRow(sa_size_t n, DataTableRow& row) const
{
  if ( row.Size() != NCols() )
    throw SzMismatchError(" BaseDataTable::GetRow(n, row) - row.Size() != NCols() ");
  GetRow(n, row.MTVPtr());
  return row;
}

/*!
  For thread-safe operation, specify a valid \b mtvp  pointer (!= NULL)  
*/
MuTyV* BaseDataTable::GetRow(sa_size_t n, MuTyV* mtvp) const 
{
  DT_TSOP_SYNC dttss(mThS); dttss.NOp();  // Thread-safe operation 

  if ((n < 0) || (n >= NEntry())) 
    throw RangeCheckError("BaseDataTable::GetRow() out of range line index n");
  if (mtvp == NULL) {
    if (mVarMTV == NULL) mVarMTV = new MuTyV[NVar()];
    mtvp = mVarMTV;
  }
  sa_size_t bid = n/mSegSz;
  sa_size_t off = n%mSegSz;
  for(sa_size_t k=0; k<mIColsP.size(); k++) 
    mtvp[mIColIdx[k]] = mIColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mLColsP.size(); k++) 
    mtvp[mLColIdx[k]] = mLColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mFColsP.size(); k++) 
    mtvp[mFColIdx[k]] = mFColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mDColsP.size(); k++) {
    if (GetColumType(mDColIdx[k]) == DateTimeField) 
      mtvp[mDColIdx[k]] = TimeStamp(mDColsP[k]->GetCstSegment(bid)[off]);
    else mtvp[mDColIdx[k]] = mDColsP[k]->GetCstSegment(bid)[off];
  }
  for(sa_size_t k=0; k<mYColsP.size(); k++) 
    mtvp[mYColIdx[k]] = mYColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mZColsP.size(); k++) 
    mtvp[mZColIdx[k]] = mZColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mSColsP.size(); k++) 
    mtvp[mSColIdx[k]] = mSColsP[k]->GetCstSegment(bid)[off];

  return mtvp;
}

#define BADVAL -1.e39

TVector<r_8> BaseDataTable::GetColumnD(sa_size_t k) const
{
  DT_TSOP_SYNC dttss(mThS); dttss.NOp();  // Thread-safe operation 

  if ((k < 0) || (k >= NVar())) 
    throw RangeCheckError("BaseDataTable::GetColumnD() out of range column index k");
  sa_size_t sk = mNames[k].ser;
  sa_size_t i = 0; 
  TVector<r_8> rv(NEntry());

  for (sa_size_t is=0; is<NbSegments(); is++) {
    switch (mNames[k].type) {
    case IntegerField :
      for(sa_size_t j=0; (j<SegmentSize())&&(i<NEntry()); j++,i++)
        rv(i) = mIColsP[sk]->GetCstSegment(is)[j];
      break;
    case LongField :
      for(sa_size_t j=0; (j<SegmentSize())&&(i<NEntry()); j++,i++)
        rv(i) = mLColsP[sk]->GetCstSegment(is)[j];
      break;
    case FloatField :
      for(sa_size_t j=0; (j<SegmentSize())&&(i<NEntry()); j++,i++)
        rv(i) = mFColsP[sk]->GetCstSegment(is)[j];
      break;
    case DoubleField :
    case DateTimeField :
      for(sa_size_t j=0; (j<SegmentSize())&&(i<NEntry()); j++,i++)
        rv(i) = mDColsP[sk]->GetCstSegment(is)[j];
      break;
    case ComplexField :
      for(sa_size_t j=0; (j<SegmentSize())&&(i<NEntry()); j++,i++)
        rv(i) = mYColsP[sk]->GetCstSegment(is)[j].real();
      break;
    case DoubleComplexField :
      for(sa_size_t j=0; (j<SegmentSize())&&(i<NEntry()); j++,i++)
        rv(i) = mZColsP[sk]->GetCstSegment(is)[j].real();
      break;
    case StringField :
      for(sa_size_t j=0; (j<SegmentSize())&&(i<NEntry()); j++,i++)
        rv = atof(mSColsP[sk]->GetCstSegment(is)[j].c_str());
      break;
    default:
      for(sa_size_t j=0; (j<SegmentSize())&&(i<NEntry()); j++,i++)  
        rv(i) = BADVAL;
      break;
    }
  }
  return rv ; 
}

void BaseDataTable::CopyMerge(BaseDataTable const& a, bool cp)
{
  if (cp && (NEntry() > 0) )
    throw ParmError("BaseDataTable::CopyMerge(a) Table has entries already");
  if (a.NVar() == 0)  throw ParmError("BaseDataTable::CopyMerge(a) Table a has no column");
  if (NVar() == 0) CopyStructure(a);
  else if (!CompareStructure(a)) 
    throw SzMismatchError("BaseDataTable::CopyMerge(a) (this,a) have different table structure");
  if (a.NEntry() == 0) {
    cout << " BaseDataTable::CopyMerge(a)/Warning : table a has zero (0) entry ! " << endl;
    return;
  }
  for(sa_size_t kk=0; kk<a.NEntry(); kk++) 
    AddRow(a.GetLine(kk));
}


//! Returns the associated DVList object
DVList&  BaseDataTable::Info() const
{
  if (mInfo == NULL)  mInfo = new DVList;
  return(*mInfo);
}

/*!
  Formatted (text) output of the table, for lines lstart <= l_index < lend , with step lstep
  \param os : output stream (formatted output)
  \param lstart : start row (line) index
  \param lend : end row (line) index 
  \param lstep : row index increment
*/
ostream& BaseDataTable::Print(ostream& os, sa_size_t lstart, sa_size_t lend, sa_size_t lstep) const
{
  os << "##### BaseDataTable::Print() - Table(NRow=" << NEntry() << " , NCol=" 
     << NVar() << ") ##### " << endl;
  os << "#! " ;
  for (size_t i=0; i<NVar(); i++) {
    string nom = mNames[i].nom;
    nom += ':'; nom += ColTypeToString(mNames[i].type);
    os << setw(12) << nom << " ";
  }
  os << endl;
  os << "##########################################################################" << endl;
  for (sa_size_t l=lstart; l<lend; l+=lstep)
    os << TableRowToString(l, true) << endl;
  return os;
}

/*! In addition to printing the number of entries and column names, 
  this method prints also minimum/maximum value for each column. 
  This information might be computed when the Show() method is called.
  This may take some time for tables with large number of entries (>~ 10^6)
*/
void   BaseDataTable::Show(ostream& os) const 
{
  os << "BaseDataTable: NVar= " << NVar() << " NEnt= " << NEntry() 
     << " ( SegSize= " << SegmentSize() << "  NbSegments= " 
     << NbSegments() << " )" << endl;
  os << "--------------------------------------------------------------------" << endl;
  os << setw(3) << "i" << ":" << setw(20) << " Name" << " ("  
     << setw(4) << "Type" << ") | " 
     << setw(15) << " Min " << " | " << setw(15) << " Max " << endl;
  os << "--------------------------------------------------------------------" << endl;
  r_8 min, max ;
  for(sa_size_t i = 0 ; i < NVar() ; i++) {
    GetMinMax(i, min, max) ;
    string nom;
    if (mNames[i].nom.length() > 20) 
      nom = mNames[i].nom.substr(20);
    else nom = mNames[i].nom;
    os << setw(3) << i << ":" << setw(20) << nom << " (" 
       << setw(4) << ColTypeToString(mNames[i].type) << ") | " 
       << setw(15) << min << " | " << setw(15) << max << endl;
  }
  os << "--------------------------------------------------------------------" << endl;
  return;
}

//! Fills table from an ascii (text) file
/*
  Return number of non empt lines (added to table)
  \param is : input ascii (text) stream
  \param clm : Lines starting with clm are ignored (comments)
  \param sep : separator between different fields (columns)
*/
sa_size_t  BaseDataTable::FillFromASCIIFile(istream& is, char clm, const char* sep)
{
  string str;
  if (mVarMTV == NULL) mVarMTV = new MuTyV[NVar()];
  sa_size_t iv, nl;
  nl = 0;
  while (!is.eof()) {
    str = "";
    getline(is, str);
    if (is.good() || is.eof()) {
      size_t l = str.length();
      if ((l == 0) || (str[0]==clm)) continue;
      for(iv=0; iv<NVar(); iv++) mVarMTV[iv] = 0.;
      iv = 0;
      size_t q = 0;
      size_t p = 0;
      while ( (q < l) && (iv < NVar()) ) {
        p = str.find_first_not_of(sep,q);
        if (p >= l)  break;
        if (str[p] == '\'')    {  // Decodage d'un string
          q = str.find('\'',p+1);
          if (q < l)  { 
            mVarMTV[iv] = str.substr(p+1,q-p-1);
            q++;
          }
          else mVarMTV[iv] = str.substr(p+1,l-p-1);
          iv++;
        }
        else {
          q = str.find_first_of(sep,p);
          if (q > l) q = l;
          mVarMTV[iv] = str.substr(p,q-p);
          iv++;
        }
        if (mNames[iv-1].type == DateTimeField) {
          string tts = (string)mVarMTV[iv-1];
          mVarMTV[iv-1] = TimeStamp(tts);
        }
      }
      AddRow(mVarMTV);
      nl++;
    }
  }  // Fin boucle lignes fichier
  cout << "BaseDataTable::FillFromASCIIFile()/Info: " << nl << " lines decoded from stream " << endl; 
  return(nl);
}



// 
// ------------------------------------
// ------- Interface NTuple -----------
// ------------------------------------
// 
sa_size_t BaseDataTable::NbLines() const
{
  return(NEntry());
}

sa_size_t BaseDataTable::NbColumns() const
{
  return(NVar());
}

r_8* BaseDataTable::GetLineD(sa_size_t n) const 
{
  if ((n < 0) || (n >= NEntry())) 
    throw RangeCheckError("BaseDataTable::GetLineD() out of range line index n");
  if (mVarD == NULL) mVarD = new r_8[NVar()];

  sa_size_t bid = n/mSegSz;
  sa_size_t off = n%mSegSz;
  for(sa_size_t k=0; k<mIColsP.size(); k++) 
    mVarD[mIColIdx[k]] = mIColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mLColsP.size(); k++) 
    mVarD[mLColIdx[k]] = mLColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mFColsP.size(); k++) 
    mVarD[mFColIdx[k]] = mFColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mDColsP.size(); k++) 
    mVarD[mDColIdx[k]] = mDColsP[k]->GetCstSegment(bid)[off];
  for(sa_size_t k=0; k<mYColsP.size(); k++) 
    mVarD[mYColIdx[k]] = mYColsP[k]->GetCstSegment(bid)[off].real();
  for(sa_size_t k=0; k<mZColsP.size(); k++) 
    mVarD[mZColIdx[k]] = mZColsP[k]->GetCstSegment(bid)[off].real();
  for(sa_size_t k=0; k<mSColsP.size(); k++) 
    mVarD[mSColIdx[k]] = atof(mSColsP[k]->GetCstSegment(bid)[off].c_str());

  return mVarD;
}


r_8  BaseDataTable::GetCell(sa_size_t n, sa_size_t k) const 
{
  if ((n < 0) || (n >= NEntry())) 
    throw RangeCheckError("BaseDataTable::GetCell() out of range line index n");
  if ((k < 0) || (k >= NVar())) 
    throw RangeCheckError("BaseDataTable::GetCell() out of range column index k");
  double rv = BADVAL;
  sa_size_t sk = mNames[k].ser;
  sa_size_t bid = n/mSegSz;
  sa_size_t off = n%mSegSz;

  switch (mNames[k].type) {
  case IntegerField :
    rv = mIColsP[sk]->GetCstSegment(bid)[off];
    break;
  case LongField :
    rv = mLColsP[sk]->GetCstSegment(bid)[off];
    break;
  case FloatField :
    rv = mFColsP[sk]->GetCstSegment(bid)[off];
    break;
  case DoubleField :
  case DateTimeField :
    rv = mDColsP[sk]->GetCstSegment(bid)[off];
    break;
  case ComplexField :
    rv = mYColsP[sk]->GetCstSegment(bid)[off].real();
    break;
  case DoubleComplexField :
    rv = mZColsP[sk]->GetCstSegment(bid)[off].real();
    break;
  case StringField :
    rv = atof(mSColsP[sk]->GetCstSegment(bid)[off].c_str());
    break;
  default:
    rv = BADVAL;
    break;
  }
  return rv ; 
}


r_8  BaseDataTable::GetCell(sa_size_t n, string const& nom) const 
{
  return GetCell(n, IndexNom(nom));
}

string   BaseDataTable::GetCelltoString(sa_size_t n, sa_size_t k) const
{
  if ((n < 0) || (n >= NEntry())) 
    throw RangeCheckError("BaseDataTable::GetCell() out of range line index n");
  if ((k < 0) || (k >= NVar())) 
    throw RangeCheckError("BaseDataTable::GetCell() out of range column index k");
  MuTyV rv;;
  sa_size_t sk = mNames[k].ser;
  sa_size_t bid = n/mSegSz;
  sa_size_t off = n%mSegSz;
  switch (mNames[k].type) {
  case IntegerField :
    rv = mIColsP[sk]->GetCstSegment(bid)[off];
    break;
  case LongField :
    rv = mLColsP[sk]->GetCstSegment(bid)[off];
    break;
  case FloatField :
    rv = mFColsP[sk]->GetCstSegment(bid)[off];
    break;
  case DoubleField :
    rv = mDColsP[sk]->GetCstSegment(bid)[off];
    break;
  case ComplexField :
    rv = mYColsP[sk]->GetCstSegment(bid)[off];
    break;
  case DoubleComplexField :
    rv = mZColsP[sk]->GetCstSegment(bid)[off];
    break;
  case StringField :
    rv = mSColsP[sk]->GetCstSegment(bid)[off];
    break;
  case DateTimeField :
    rv = TimeStamp(mDColsP[sk]->GetCstSegment(bid)[off]);
    break;
  default:
    rv = " ";
    break;
  }
  return (string)rv ; 
}

void BaseDataTable::GetMinMax(sa_size_t k, double& min, double& max)  const
{
  min = 9E39 ; max = -9E39 ; 
  if ((k < 0) || (k >= NVar())) 
    throw RangeCheckError("BaseDataTable::GetCell() out of range column index k");
  if (mMinMaxNEnt.size() < NVar()) {
    mMin.clear();
    mMax.clear();
    mMinMaxNEnt.clear();
    for(size_t kk=0; kk<NVar(); kk++) {
      mMin.push_back(0.);
      mMax.push_back(0.);
      mMinMaxNEnt.push_back(0);
    }
  }
  if (mMinMaxNEnt[k] == mNEnt) {
    min = mMin[k];
    max = mMax[k];
    return;
  }
  sa_size_t sk = mNames[k].ser;

  sa_size_t cnt = 0;
  switch (mNames[k].type) {
  case IntegerField :
    for(size_t is=0; is<mIColsP[sk]->NbSegments(); is++) {
      const int_4* sp = mIColsP[sk]->GetCstSegment(is);
      for(size_t n=0; n<mIColsP[sk]->SegmentSize(); n++) {
        if (cnt >= NEntry())  break;
        if (sp[n] > max) max = sp[n];
        if (sp[n] < min) min = sp[n];
        cnt++;
      }
    }
    
    break;
  case LongField :
    for(size_t is=0; is<mLColsP[sk]->NbSegments(); is++) {
      const int_8* sp = mLColsP[sk]->GetCstSegment(is);
      for(size_t n=0; n<mLColsP[sk]->SegmentSize(); n++) {
        if (cnt >= NEntry())  break;
        if (sp[n] > max) max = sp[n];
        if (sp[n] < min) min = sp[n];
        cnt++;
      }
    }
    break;
  case FloatField :
    for(size_t is=0; is<mFColsP[sk]->NbSegments(); is++) {
      const r_4* sp = mFColsP[sk]->GetCstSegment(is);
      for(size_t n=0; n<mFColsP[sk]->SegmentSize(); n++) {
        if (cnt >= NEntry())  break;
        if (sp[n] > max) max = sp[n];
        if (sp[n] < min) min = sp[n];
        cnt++;
      }
    }
    break;
  case DoubleField :
  case DateTimeField :
    for(size_t is=0; is<mDColsP[sk]->NbSegments(); is++) {
      const r_8* sp = mDColsP[sk]->GetCstSegment(is);
      for(size_t n=0; n<mDColsP[sk]->SegmentSize(); n++) {
        if (cnt >= NEntry())  break;
        if (sp[n] > max) max = sp[n];
        if (sp[n] < min) min = sp[n];
        cnt++;
      }
    }
    break;
  case ComplexField :
    for(size_t is=0; is<mYColsP[sk]->NbSegments(); is++) {
      const complex<r_4> * sp = mYColsP[sk]->GetCstSegment(is);
      for(size_t n=0; n<mYColsP[sk]->SegmentSize(); n++) {
        if (cnt >= NEntry())  break;
        if (sp[n].real() > max) max = sp[n].real();
        if (sp[n].real() < min) min = sp[n].real();
        cnt++;
      }
    }
    break;
  case DoubleComplexField :
    for(size_t is=0; is<mZColsP[sk]->NbSegments(); is++) {
      const complex<r_8> * sp = mZColsP[sk]->GetCstSegment(is);
      for(size_t n=0; n<mZColsP[sk]->SegmentSize(); n++) {
        if (cnt >= NEntry())  break;
        if (sp[n].real() > max) max = sp[n].real();
        if (sp[n].real() < min) min = sp[n].real();
        cnt++;
      }
    }
    break;
  case StringField :
    return;
    break;
  default:
    return;
    break;
  }  

  mMinMaxNEnt[k] = cnt; 
  mMin[k] = min;
  mMax[k] = max;
  return ; 
}


void BaseDataTable::GetMinMax(string const & nom, double& min, double& max)   const
{
  GetMinMax(IndexNom(nom), min, max) ; 
}


sa_size_t BaseDataTable::ColumnIndex(string const& nom) const 
{
  return IndexNom(nom) ; 
}


string BaseDataTable::ColumnName(sa_size_t k) const 
{
  return NomIndex(k) ; 
}


string BaseDataTable::VarList_C(const char* nomx) const 
{
  string rets="";
  sa_size_t i;
  for(i=0; i<NVar(); i++) {
    if ( (i%5 == 0) && (i > 0) )  rets += ";";  
    if (i%5 == 0)   rets += "\ndouble "; 
    else rets += ",";
    rets += mNames[i].nom;
  }
  rets += "; \n";
  if (nomx) { 
    char buff[256];
    for(i=0; i<NVar(); i++) {
      rets += mNames[i].nom;
      rets += '=';
      
      sprintf(buff,"%s[%ld]; ",  nomx, (long)i);
      rets += buff;
      if ( (i%3 == 0) && (i > 0) )  rets += "\n"; 
    }
  }
  return(rets);
}


string BaseDataTable::LineHeaderToString() const 
{
  string rets,s;
  
  for(int i=0; i<NVar(); i++) {
    s = mNames[i].nom;
    size_t l = s.length();
    for(size_t ii=l; ii<12; ii++) s += ' ';
    if (i > 0) rets += ' ';   
    rets += s;
  }
  return(rets);  
}

/*!
  Return a table row (line) as a string
  \sa TableRowToString()
*/
string BaseDataTable::LineToString(sa_size_t n) const
{
  return TableRowToString(n, false);
}

/*!
  \param n : table row index ( 0 ... NEntry()-1) 
  \param qstr : if true , enclose strings in quotes ''
  \param sep : separates fields using \b sep
  \param fw : minimum field width
 */
string BaseDataTable::TableRowToString(sa_size_t n, bool qstr, 
                                       const char* sep, int fw) const 
{
  if ((n < 0) || (n >= NEntry())) 
    throw RangeCheckError("BaseDataTable::GetCell() out of range line index n");
  string rs;
  MuTyV rv;;
  sa_size_t bid = n/mSegSz;
  sa_size_t off = n%mSegSz;
  for(sa_size_t k=0; k<NVar(); k++) {
    sa_size_t sk = mNames[k].ser;
    switch (mNames[k].type) {
    case IntegerField :
      rv = mIColsP[sk]->GetCstSegment(bid)[off];
      break;
    case LongField :
      rv = mLColsP[sk]->GetCstSegment(bid)[off];
      break;
    case FloatField :
      rv = mFColsP[sk]->GetCstSegment(bid)[off];
      break;
    case DoubleField :
      rv = mDColsP[sk]->GetCstSegment(bid)[off];
      break;
    case ComplexField :
      rv = mYColsP[sk]->GetCstSegment(bid)[off];
      break;
    case DoubleComplexField :
      rv = mZColsP[sk]->GetCstSegment(bid)[off];
      break;
    case StringField :
      rv = mSColsP[sk]->GetCstSegment(bid)[off];
      break;
    case DateTimeField :
      rv = TimeStamp(mDColsP[sk]->GetCstSegment(bid)[off]);
      break;
    default:
      rv = " ";
      break;
    }
    string s;
    if ( (mNames[k].type == StringField) && (qstr) ) {
      s = '\'';  s += (string)rv; s += '\''; 
    }
    else  s= (string)rv;
    size_t l = s.length();
    for(size_t ii=l; ii<fw; ii++) s += ' ';
    if (k > 0) rs += sep;
    rs += s;
  }
  return rs;
}

} // FIN namespace SOPHYA