tokenizer.cpp 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271
  1. ///////////////////////////////////////////////////////////////////////////////
  2. // Name: tests/strings/strings.cpp
  3. // Purpose: wxStringTokenizer unit test
  4. // Author: Vadim Zeitlin
  5. // Created: 2005-12-20 (extacted from strings.cpp)
  6. // Copyright: (c) 2004-2005 Vadim Zeitlin
  7. ///////////////////////////////////////////////////////////////////////////////
  8. // ----------------------------------------------------------------------------
  9. // headers
  10. // ----------------------------------------------------------------------------
  11. #include "testprec.h"
  12. #ifdef __BORLANDC__
  13. #pragma hdrstop
  14. #endif
  15. #ifndef WX_PRECOMP
  16. #include "wx/wx.h"
  17. #endif // WX_PRECOMP
  18. #include "wx/tokenzr.h"
  19. // ----------------------------------------------------------------------------
  20. // test class
  21. // ----------------------------------------------------------------------------
  22. class TokenizerTestCase : public CppUnit::TestCase
  23. {
  24. public:
  25. TokenizerTestCase() { }
  26. private:
  27. CPPUNIT_TEST_SUITE( TokenizerTestCase );
  28. CPPUNIT_TEST( GetCount );
  29. CPPUNIT_TEST( GetPosition );
  30. CPPUNIT_TEST( GetString );
  31. CPPUNIT_TEST( LastDelimiter );
  32. CPPUNIT_TEST( StrtokCompat );
  33. CPPUNIT_TEST_SUITE_END();
  34. void GetCount();
  35. void GetPosition();
  36. void GetString();
  37. void LastDelimiter();
  38. void StrtokCompat();
  39. DECLARE_NO_COPY_CLASS(TokenizerTestCase)
  40. };
  41. // register in the unnamed registry so that these tests are run by default
  42. CPPUNIT_TEST_SUITE_REGISTRATION( TokenizerTestCase );
  43. // also include in its own registry so that these tests can be run alone
  44. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( TokenizerTestCase, "TokenizerTestCase" );
  45. // ----------------------------------------------------------------------------
  46. // test data
  47. // ----------------------------------------------------------------------------
  48. static const struct TokenizerTestData
  49. {
  50. // the string to tokenize
  51. const wxChar *str;
  52. // the delimiters to use
  53. const wxChar *delims;
  54. // the tokenizer mode
  55. wxStringTokenizerMode mode;
  56. // expected number of tokens
  57. size_t count;
  58. }
  59. gs_testData[] =
  60. {
  61. { wxT(""), wxT(" "), wxTOKEN_DEFAULT, 0 },
  62. { wxT(""), wxT(" "), wxTOKEN_RET_EMPTY, 0 },
  63. { wxT(""), wxT(" "), wxTOKEN_RET_EMPTY_ALL, 0 },
  64. { wxT(""), wxT(" "), wxTOKEN_RET_DELIMS, 0 },
  65. { wxT(":"), wxT(":"), wxTOKEN_RET_EMPTY, 1 },
  66. { wxT(":"), wxT(":"), wxTOKEN_RET_DELIMS, 1 },
  67. { wxT(":"), wxT(":"), wxTOKEN_RET_EMPTY_ALL, 2 },
  68. { wxT("::"), wxT(":"), wxTOKEN_RET_EMPTY, 1 },
  69. { wxT("::"), wxT(":"), wxTOKEN_RET_DELIMS, 1 },
  70. { wxT("::"), wxT(":"), wxTOKEN_RET_EMPTY_ALL, 3 },
  71. { wxT("Hello, world"), wxT(" "), wxTOKEN_DEFAULT, 2 },
  72. { wxT("Hello, world "), wxT(" "), wxTOKEN_DEFAULT, 2 },
  73. { wxT("Hello, world"), wxT(","), wxTOKEN_DEFAULT, 2 },
  74. { wxT("Hello, world!"), wxT(",!"), wxTOKEN_DEFAULT, 2 },
  75. { wxT("Hello,, world!"), wxT(",!"), wxTOKEN_DEFAULT, 3 },
  76. { wxT("Hello,, world!"), wxT(",!"), wxTOKEN_STRTOK, 2 },
  77. { wxT("Hello, world!"), wxT(",!"), wxTOKEN_RET_EMPTY_ALL, 3 },
  78. { wxT("username:password:uid:gid:gecos:home:shell"),
  79. wxT(":"), wxTOKEN_DEFAULT, 7 },
  80. { wxT("1:2::3:"), wxT(":"), wxTOKEN_DEFAULT, 4 },
  81. { wxT("1:2::3:"), wxT(":"), wxTOKEN_RET_EMPTY, 4 },
  82. { wxT("1:2::3:"), wxT(":"), wxTOKEN_RET_EMPTY_ALL, 5 },
  83. { wxT("1:2::3:"), wxT(":"), wxTOKEN_RET_DELIMS, 4 },
  84. { wxT("1:2::3:"), wxT(":"), wxTOKEN_STRTOK, 3 },
  85. { wxT("1:2::3::"), wxT(":"), wxTOKEN_DEFAULT, 4 },
  86. { wxT("1:2::3::"), wxT(":"), wxTOKEN_RET_EMPTY, 4 },
  87. { wxT("1:2::3::"), wxT(":"), wxTOKEN_RET_EMPTY_ALL, 6 },
  88. { wxT("1:2::3::"), wxT(":"), wxTOKEN_RET_DELIMS, 4 },
  89. { wxT("1:2::3::"), wxT(":"), wxTOKEN_STRTOK, 3 },
  90. { wxT("1 \t3\t4 6 "), wxDEFAULT_DELIMITERS, wxTOKEN_DEFAULT, 4 },
  91. { wxT("1 \t3\t4 6 "), wxDEFAULT_DELIMITERS, wxTOKEN_STRTOK, 4 },
  92. { wxT("1 \t3\t4 6 "), wxDEFAULT_DELIMITERS, wxTOKEN_RET_EMPTY, 6 },
  93. { wxT("1 \t3\t4 6 "), wxDEFAULT_DELIMITERS, wxTOKEN_RET_EMPTY_ALL, 9 },
  94. { wxT("01/02/99"), wxT("/-"), wxTOKEN_DEFAULT, 3 },
  95. { wxT("01-02/99"), wxT("/-"), wxTOKEN_RET_DELIMS, 3 },
  96. };
  97. // helper function returning the string showing the index for which the test
  98. // fails in the diagnostic message
  99. static std::string Nth(size_t n)
  100. {
  101. return std::string(wxString::Format(wxT("for loop index %lu"),
  102. (unsigned long)n).mb_str());
  103. }
  104. // ----------------------------------------------------------------------------
  105. // the tests
  106. // ----------------------------------------------------------------------------
  107. void TokenizerTestCase::GetCount()
  108. {
  109. for ( size_t n = 0; n < WXSIZEOF(gs_testData); n++ )
  110. {
  111. const TokenizerTestData& ttd = gs_testData[n];
  112. wxStringTokenizer tkz(ttd.str, ttd.delims, ttd.mode);
  113. CPPUNIT_ASSERT_EQUAL_MESSAGE( Nth(n), ttd.count, tkz.CountTokens() );
  114. size_t count = 0;
  115. while ( tkz.HasMoreTokens() )
  116. {
  117. tkz.GetNextToken();
  118. count++;
  119. }
  120. CPPUNIT_ASSERT_EQUAL_MESSAGE( Nth(n), ttd.count, count );
  121. }
  122. }
  123. // call this with the string to tokenize, delimeters to use and the expected
  124. // positions (i.e. results of GetPosition()) after each GetNextToken() call,
  125. // terminate positions with 0
  126. static void
  127. DoTestGetPosition(const wxChar *s, const wxChar *delims, int pos, ...)
  128. {
  129. wxStringTokenizer tkz(s, delims);
  130. CPPUNIT_ASSERT_EQUAL( (size_t)0, tkz.GetPosition() );
  131. va_list ap;
  132. va_start(ap, pos);
  133. for ( ;; )
  134. {
  135. if ( !pos )
  136. {
  137. CPPUNIT_ASSERT( !tkz.HasMoreTokens() );
  138. break;
  139. }
  140. tkz.GetNextToken();
  141. CPPUNIT_ASSERT_EQUAL( (size_t)pos, tkz.GetPosition() );
  142. pos = va_arg(ap, int);
  143. }
  144. va_end(ap);
  145. }
  146. void TokenizerTestCase::GetPosition()
  147. {
  148. DoTestGetPosition(wxT("foo"), wxT("_"), 3, 0);
  149. DoTestGetPosition(wxT("foo_bar"), wxT("_"), 4, 7, 0);
  150. DoTestGetPosition(wxT("foo_bar_"), wxT("_"), 4, 8, 0);
  151. }
  152. // helper for GetString(): the parameters are the same as for DoTestGetPosition
  153. // but it checks GetString() return value instead of GetPosition()
  154. static void
  155. DoTestGetString(const wxChar *s, const wxChar *delims, int pos, ...)
  156. {
  157. wxStringTokenizer tkz(s, delims);
  158. CPPUNIT_ASSERT_EQUAL( wxString(s), tkz.GetString() );
  159. va_list ap;
  160. va_start(ap, pos);
  161. for ( ;; )
  162. {
  163. if ( !pos )
  164. {
  165. CPPUNIT_ASSERT( tkz.GetString().empty() ) ;
  166. break;
  167. }
  168. tkz.GetNextToken();
  169. CPPUNIT_ASSERT_EQUAL( wxString(s + pos), tkz.GetString() );
  170. pos = va_arg(ap, int);
  171. }
  172. va_end(ap);
  173. }
  174. void TokenizerTestCase::GetString()
  175. {
  176. DoTestGetString(wxT("foo"), wxT("_"), 3, 0);
  177. DoTestGetString(wxT("foo_bar"), wxT("_"), 4, 7, 0);
  178. DoTestGetString(wxT("foo_bar_"), wxT("_"), 4, 8, 0);
  179. }
  180. void TokenizerTestCase::LastDelimiter()
  181. {
  182. wxStringTokenizer tkz(wxT("a+-b=c"), wxT("+-="));
  183. tkz.GetNextToken();
  184. CPPUNIT_ASSERT_EQUAL( wxT('+'), tkz.GetLastDelimiter() );
  185. tkz.GetNextToken();
  186. CPPUNIT_ASSERT_EQUAL( wxT('-'), tkz.GetLastDelimiter() );
  187. tkz.GetNextToken();
  188. CPPUNIT_ASSERT_EQUAL( wxT('='), tkz.GetLastDelimiter() );
  189. tkz.GetNextToken();
  190. CPPUNIT_ASSERT_EQUAL( wxT('\0'), tkz.GetLastDelimiter() );
  191. }
  192. void TokenizerTestCase::StrtokCompat()
  193. {
  194. for ( size_t n = 0; n < WXSIZEOF(gs_testData); n++ )
  195. {
  196. const TokenizerTestData& ttd = gs_testData[n];
  197. if ( ttd.mode != wxTOKEN_STRTOK )
  198. continue;
  199. #if wxUSE_UNICODE
  200. wxWCharBuffer
  201. #else
  202. wxCharBuffer
  203. #endif
  204. buf(ttd.str);
  205. wxChar *last;
  206. wxChar *s = wxStrtok(buf.data(), ttd.delims, &last);
  207. wxStringTokenizer tkz(ttd.str, ttd.delims, ttd.mode);
  208. while ( tkz.HasMoreTokens() )
  209. {
  210. CPPUNIT_ASSERT_EQUAL( wxString(s), tkz.GetNextToken() );
  211. s = wxStrtok(NULL, ttd.delims, &last);
  212. }
  213. }
  214. }