Planet
navi homePPSaboutscreenshotsdownloaddevelopmentforum

source: downloads/boost_1_33_1/libs/tokenizer/tokenizer.htm @ 20

Last change on this file since 20 was 12, checked in by landauf, 18 years ago

added boost
File size: 5.4 KB
Line 
1<html>
2
3<head>
4<meta http-equiv="Content-Type"
5content="text/html; charset=iso-8859-1">
6<meta name="GENERATOR" content="Microsoft FrontPage Express 2.0">
7<title>Boost Tokenizer Class</title>
8<!--
9  -- Copyright © John Bandela 2001
10  --
11  -- Permission to use, copy, modify, distribute and sell this software
12  -- and its documentation for any purpose is hereby granted without fee,
13  -- provided that the above copyright notice appears in all copies and
14  -- that both that copyright notice and this permission notice appear
15  -- in supporting documentation.  Jeremy Siek makes no
16  -- representations about the suitability of this software for any
17  -- purpose.  It is provided "as is" without express or implied warranty.
18  -->
19</head>
20
21<body bgcolor="#FFFFFF" text="#000000" link="#0000EE"
22vlink="#551A8B" alink="#FF0000">
23
24<p><img src="../../boost.png" alt="C++ Boost" width="277"
25height="86"> <br>
26</p>
27
28<h1 align="center">Tokenizer Class</h1>
29
30<pre>  template &lt;
31        class TokenizerFunc = char_delimiters_separator&lt;char&gt;,
32        class Iterator = std::string::const_iterator,
33        class Type = std::string
34  &gt;
35  class tokenizer
36</pre>
37
38<p>The tokenizer class provides a container view of a series of
39tokens contained in a sequence. You set the sequence to parse and
40the TokenizerFunction to use to parse the sequence either upon
41construction or using the assign member function. Note: No
42parsing is actually done upon construction. Parsing is done on
43demand as the tokens are accessed via the iterator provided by
44begin.</p>
45
46<h2>Example</h2>
47
48<pre>// simple_example_1.cpp
49#include&lt;iostream&gt;
50#include&lt;boost/tokenizer.hpp&gt;
51#include&lt;string&gt;
52
53int main(){
54   using namespace std;
55   using namespace boost;
56   string s = &quot;This is,  a test&quot;;
57   tokenizer&lt;&gt; tok(s);
58   for(tokenizer&lt;&gt;::iterator beg=tok.begin(); beg!=tok.end();++beg){
59       cout &lt;&lt; *beg &lt;&lt; &quot;\n&quot;;
60   }
61}
62</pre>
63
64<p>&nbsp;</p>
65
66<h3>Template Parameters</h3>
67
68<table border="1">
69    <tr>
70        <th>Parameter</th>
71        <th>Description</th>
72    </tr>
73    <tr>
74        <td><tt>TokenizerFunc</tt></td>
75        <td>The TokenizerFunction used to parse the sequence.</td>
76    </tr>
77    <tr>
78        <td><tt>Iterator</tt></td>
79        <td>The type of the iterator the specifies the sequence.</td>
80    </tr>
81    <tr>
82        <td><tt>Type</tt></td>
83        <td>The type of the token, typically string.</td>
84    </tr>
85</table>
86
87<p>&nbsp;</p>
88
89<h2>Related Types</h2>
90
91<table border="1">
92    <tr>
93        <td><p align="center"><strong>Type</strong></p>
94        </td>
95        <td><p align="center"><strong>Remarks</strong></p>
96        </td>
97    </tr>
98    <tr>
99        <td>iterator</td>
100        <td>The type returned by begin and end. Note: the
101        category of iterator will be at most ForwardIterator. It
102        will be InputIterator if the Iterator template parameter
103        is an InputIterator. For any other category, it will be
104        ForwardIterator.</td>
105    </tr>
106    <tr>
107        <td>const_iterator</td>
108        <td>Same type as iterator.</td>
109    </tr>
110    <tr>
111        <td>value_type</td>
112        <td>Same type as the template parameter Type</td>
113    </tr>
114    <tr>
115        <td>reference</td>
116        <td>Same type as value_type&amp;</td>
117    </tr>
118    <tr>
119        <td>const_reference</td>
120        <td>Same type as const reference</td>
121    </tr>
122    <tr>
123        <td>pointer</td>
124        <td>Same type as value_type*</td>
125    </tr>
126    <tr>
127        <td>const_pointer</td>
128        <td>Same type as const pointer</td>
129    </tr>
130    <tr>
131        <td>size_type</td>
132        <td>void</td>
133    </tr>
134    <tr>
135        <td>difference_type</td>
136        <td>void</td>
137    </tr>
138</table>
139
140<p>&nbsp;</p>
141
142<h2>Construction and Member Functions</h2>
143
144<pre>tokenizer(Iterator first, Iterator last,const TokenizerFunc&amp; f = TokenizerFunc())
145
146template&lt;class Container&gt;
147tokenizer(const Container&amp; c,const TokenizerFunc&amp; f = TokenizerFunc())
148
149void assign(Iterator first, Iterator last)
150
151void assign(Iterator first, Iterator last, const TokenizerFunc&amp; f)
152
153template&lt;class Container&gt;
154void assign(const Container&amp; c)
155
156template&lt;class Container&gt;
157void assign(const Container&amp; c, const TokenizerFunc&amp; f)
158
159iterator begin() const
160
161iterator end() const
162</pre>
163
164<table border="1">
165    <tr>
166        <td><p align="center"><strong>Parameter</strong></p>
167        </td>
168        <td><p align="center"><strong>Description</strong></p>
169        </td>
170    </tr>
171    <tr>
172        <td>c</td>
173        <td>A container that contains the sequence to parse. Note:
174        c.begin() and c.end() must be convertible to the template
175        parameter Iterator.</td>
176    </tr>
177    <tr>
178        <td>f</td>
179        <td>A functor that is a model of TokenizerFunction that
180        will be used to parse the sequence.</td>
181    </tr>
182    <tr>
183        <td>first</td>
184        <td>The iterator that represents the beginning position
185        in the sequence to be parsed.</td>
186    </tr>
187    <tr>
188        <td>last</td>
189        <td>The iterator that represents the past the end
190        position in the sequence to be parsed.</td>
191    </tr>
192</table>
193
194<p>&nbsp;</p>
195
196<hr>
197
198<p>© Copyright John R. Bandela 2001. Permission to copy, use,
199modify, sell and distribute this document is granted provided
200this copyright notice appears in all copies. This document is
201provided &quot;as is&quot; without express or implied warranty,
202and with no claim as to its suitability for any purpose.</p>
203</body>
204</html>
Note: See TracBrowser for help on using the repository browser.