Return to Answer

As per suggested edit, added semicolon and std:: to string instances without it

edited Jul 13, 2015 at 6:48

9.4k
5
56
62

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; }; //CPP file const std::string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (std::string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (std::string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Example:

std::vector <std::string> v; Tokenizer s("split this string", " "); while (s.NextToken()) { v.push_back(s.GetToken()); }

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; } //CPP file const string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Example:

std::vector <std::string> v; Tokenizer s("split this string", " "); while (s.NextToken()) { v.push_back(s.GetToken()); }

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; }; //CPP file const std::string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (std::string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (std::string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Example:

std::vector <std::string> v; Tokenizer s("split this string", " "); while (s.NextToken()) { v.push_back(s.GetToken()); }

deleted 12 characters in body

Source Link

edited Sep 14, 2008 at 20:28

vzczc

9.4k
5
56
62

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; } //CPP file const string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Example:

std::vector <std::string> v; conversion::Tokenizer s("split this string", " "); while (s.NextToken()) { v.push_back(s.GetToken()); }

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; } //CPP file const string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Example:

std::vector <std::string> v; conversion::Tokenizer s("split this string", " "); while (s.NextToken()) { v.push_back(s.GetToken()); }

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; } //CPP file const string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Example:

std::vector <std::string> v; Tokenizer s("split this string", " "); while (s.NextToken()) { v.push_back(s.GetToken()); }

Added sample

Source Link

edited Sep 10, 2008 at 12:30

vzczc

9.4k
5
56
62

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; } //CPP file const string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Example:

std::vector <std::string> v; conversion::Tokenizer s("split this string", " "); while (s.NextToken()) { v.push_back(s.GetToken()); }

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; } //CPP file const string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Here is a sample tokenizer class that might do what you want

//Header file class Tokenizer { public: static const std::string DELIMITERS; Tokenizer(const std::string& str); Tokenizer(const std::string& str, const std::string& delimiters); bool NextToken(); bool NextToken(const std::string& delimiters); const std::string GetToken() const; void Reset(); protected: size_t m_offset; const std::string m_string; std::string m_token; std::string m_delimiters; } //CPP file const string Tokenizer::DELIMITERS(" \t\n\r"); Tokenizer::Tokenizer(const std::string& s) : m_string(s), m_offset(0), m_delimiters(DELIMITERS) {} Tokenizer::Tokenizer(const std::string& s, const std::string& delimiters) : m_string(s), m_offset(0), m_delimiters(delimiters) {} bool Tokenizer::NextToken() { return NextToken(m_delimiters); } bool Tokenizer::NextToken(const std::string& delimiters) { size_t i = m_string.find_first_not_of(delimiters, m_offset); if (string::npos == i) { m_offset = m_string.length(); return false; } size_t j = m_string.find_first_of(delimiters, i); if (string::npos == j) { m_token = m_string.substr(i); m_offset = m_string.length(); return true; } m_token = m_string.substr(i, j - i); m_offset = j; return true; }

Example:

std::vector <std::string> v; conversion::Tokenizer s("split this string", " "); while (s.NextToken()) { v.push_back(s.GetToken()); }

Source Link

answered Sep 10, 2008 at 12:18

vzczc

9.4k
5
56
62

Collectives™ on Stack Overflow

Return to Answer