snipt

Ctrl+h for KB shortcuts

C++

Tokenize a string in C++

#include <string>
#include <sstream>
#include <algorithm>
#include <iterator>
#include <vector>
using namespace std;

int main() {
	string sentence = "All of this has happened before...";
	
	vector<string> tokens;
	istringstream iss(sentence);
	copy(istream_iterator<string>(iss), istream_iterator<string>(), back_inserter<vector<string> >(tokens));
}
https://snipt.net/embed/fe8b7af8c6edb8d81c1483e49f03513f/
/raw/fe8b7af8c6edb8d81c1483e49f03513f/
fe8b7af8c6edb8d81c1483e49f03513f
cpp
C++
14
2019-07-21T02:20:23
True
False
False
/api/public/snipt/8262/
tokenize-a-string-in-c
<table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><a href="#L-1"> 1</a> <a href="#L-2"> 2</a> <a href="#L-3"> 3</a> <a href="#L-4"> 4</a> <a href="#L-5"> 5</a> <a href="#L-6"> 6</a> <a href="#L-7"> 7</a> <a href="#L-8"> 8</a> <a href="#L-9"> 9</a> <a href="#L-10">10</a> <a href="#L-11">11</a> <a href="#L-12">12</a> <a href="#L-13">13</a> <a href="#L-14">14</a></pre></div></td><td class="code"><div class="highlight"><pre><span></span><span id="L-1"><a name="L-1"></a><span class="cp">#include</span> <span class="cpf">&lt;string&gt;</span><span class="cp"></span> </span><span id="L-2"><a name="L-2"></a><span class="cp">#include</span> <span class="cpf">&lt;sstream&gt;</span><span class="cp"></span> </span><span id="L-3"><a name="L-3"></a><span class="cp">#include</span> <span class="cpf">&lt;algorithm&gt;</span><span class="cp"></span> </span><span id="L-4"><a name="L-4"></a><span class="cp">#include</span> <span class="cpf">&lt;iterator&gt;</span><span class="cp"></span> </span><span id="L-5"><a name="L-5"></a><span class="cp">#include</span> <span class="cpf">&lt;vector&gt;</span><span class="cp"></span> </span><span id="L-6"><a name="L-6"></a><span class="k">using</span> <span class="k">namespace</span> <span class="n">std</span><span class="p">;</span> </span><span id="L-7"><a name="L-7"></a> </span><span id="L-8"><a name="L-8"></a><span class="kt">int</span> <span class="nf">main</span><span class="p">()</span> <span class="p">{</span> </span><span id="L-9"><a name="L-9"></a> <span class="n">string</span> <span class="n">sentence</span> <span class="o">=</span> <span class="s">&quot;All of this has happened before...&quot;</span><span class="p">;</span> </span><span id="L-10"><a name="L-10"></a> </span><span id="L-11"><a name="L-11"></a> <span class="n">vector</span><span class="o">&lt;</span><span class="n">string</span><span class="o">&gt;</span> <span class="n">tokens</span><span class="p">;</span> </span><span id="L-12"><a name="L-12"></a> <span class="n">istringstream</span> <span class="n">iss</span><span class="p">(</span><span class="n">sentence</span><span class="p">);</span> </span><span id="L-13"><a name="L-13"></a> <span class="n">copy</span><span class="p">(</span><span class="n">istream_iterator</span><span class="o">&lt;</span><span class="n">string</span><span class="o">&gt;</span><span class="p">(</span><span class="n">iss</span><span class="p">),</span> <span class="n">istream_iterator</span><span class="o">&lt;</span><span class="n">string</span><span class="o">&gt;</span><span class="p">(),</span> <span class="n">back_inserter</span><span class="o">&lt;</span><span class="n">vector</span><span class="o">&lt;</span><span class="n">string</span><span class="o">&gt;</span> <span class="o">&gt;</span><span class="p">(</span><span class="n">tokens</span><span class="p">));</span> </span><span id="L-14"><a name="L-14"></a><span class="p">}</span> </span></pre></div> </td></tr></table>
c++, parsing, tokenize