source: Sophya/trunk/Poubelle/archTOI.old/tokenizer.cc@ 626

// tokenizer.cc
// Eric Aubourg         CEA/DAPNIA/SPP   septembre 1999

#include "tokenizer.h"


Tokenizer::Tokenizer(string str) 
: s(str)
{
  while (str != "") {
    size_t x = str.find_first_not_of(' ');
    if (x != string::npos) 
      str = str.substr(x);
    if (str == "") break;
    x = str.find(' ');
    if (x != string::npos) {
      string token = str.substr(0, x);
      str = str.substr(x+1);
      tokens.push_back(token);
    } else {
      tokens.push_back(str);
      break;
    }
  }
}

set<string> Tokenizer::getTokenSet() {
  return set<string>(tokens.begin(), tokens.end());
}

vector<string> Tokenizer::getTokenVector() {
  return tokens;
}